diff --git a/.asf.yaml b/.asf.yaml index 53ac5674fc405..7c1075e69dc98 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -31,6 +31,7 @@ github: features: wiki: true issues: true + discussions: true enabled_merge_buttons: # enable squash button: squash: true @@ -38,3 +39,15 @@ github: merge: false # disable rebase button: rebase: true + collaborators: + - shuwenwei + - liyuheng55555 + - CritasWang + - HxpSerein + - DanielWang2035 + - Pengzna + - Wei-hao-Li + - Colinleeo + - Caideyipi +notifications: + discussions: dev@iotdb.apache.org diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml index 5b1c3e1966745..3fc0e6360dc91 100644 --- a/.github/workflows/cluster-it-1c1d.yml +++ b/.github/workflows/cluster-it-1c1d.yml @@ -4,16 +4,16 @@ on: push: branches: - master - - 'rel/1.*' - - pipe-meta-sync + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - pipe-meta-sync + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' @@ -27,7 +27,7 @@ concurrency: env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: Simple: diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml new file mode 100644 index 0000000000000..7801c13a26a4e --- /dev/null +++ b/.github/workflows/cluster-it-1c1d1a.yml @@ -0,0 +1,63 @@ +name: Cluster IT - 1C1D1A + +on: + push: + branches: + - master + - 'rel/*' + - 'rc/*' + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - 'rc/*' + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + AINode: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Build AINode + shell: bash + run: mvn clean package -DskipTests -P with-ainode + - name: IT Test + shell: bash + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=2 \ + -pl integration-test \ + -am \ + -PAIClusterIT + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-ainode-${{ matrix.os }} + path: integration-test/target/ainode-logs + retention-days: 30 diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml index a850009e6f658..e889d3c98d20a 100644 --- a/.github/workflows/cluster-it-1c3d.yml +++ b/.github/workflows/cluster-it-1c3d.yml @@ -4,16 +4,17 @@ on: push: branches: - master - - 'rel/1.*' - - pipe-meta-sync + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - pipe-meta-sync + - 'rel/*' + - 'rc/*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' @@ -27,7 +28,7 @@ concurrency: env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: Simple: @@ -35,7 +36,7 @@ jobs: fail-fast: false max-parallel: 20 matrix: - java: [ 8, 11, 17 ] + java: [ 17 ] runs-on: [self-hosted, iotdb] # group: self-hosted # labels: iotdb diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml new file mode 100644 index 0000000000000..92522f48970d2 --- /dev/null +++ b/.github/workflows/compile-check.yml @@ -0,0 +1,53 @@ +# This workflow will compile IoTDB under jdk8 to check for compatibility issues + +name: Compile Check + +on: + push: + branches: + - master + - 'rel/*' + - 'rc/*' + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - 'rc/*' + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + compile-check: + strategy: + fail-fast: false + matrix: + java: [8] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Compiler Test + shell: bash + run: | + mvn clean package -P with-integration-tests -DskipTests -ntp diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml index 3783e563e35f8..c4f72ff0fd6fe 100644 --- a/.github/workflows/daily-it.yml +++ b/.github/workflows/daily-it.yml @@ -12,7 +12,7 @@ concurrency: env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: Simple: @@ -33,8 +33,6 @@ jobs: java-version: ${{ matrix.java }} - name: IT/UT Test shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml run: | mvn clean verify \ -P with-integration-tests \ @@ -49,3 +47,37 @@ jobs: name: cluster-log-java${{ matrix.java }}-${{ runner.os }} path: integration-test/target/cluster-logs retention-days: 3 + SingleRegionTableModel: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 8, 17 ] + runs-on: [ self-hosted, iotdb ] + # group: self-hosted + # labels: iotdb + steps: + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: IT/UT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1\ + -pl integration-test \ + -am -PTableSimpleIT + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} + path: integration-test/target/cluster-logs + retention-days: 3 \ No newline at end of file diff --git a/.github/workflows/daily-ut.yml b/.github/workflows/daily-ut.yml new file mode 100644 index 0000000000000..3ba2538fe7b1a --- /dev/null +++ b/.github/workflows/daily-ut.yml @@ -0,0 +1,57 @@ +name: Daily UT + +on: + schedule: + # Run at UTC 19:00 every day (CST 03:00 AM) + - cron: '0 19 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + unit-test: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 8 ] + os: [ ubuntu-latest, windows-latest ] + it_task: [ 'others', 'datanode' ] + include: + - java: 17 + os: macos-latest + it_task: 'datanode' + - java: 17 + os: macos-latest + it_task: 'others' + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: corretto + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Test Datanode Module with Maven + shell: bash + if: ${{ matrix.it_task == 'datanode'}} + run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true + - name: Test Other Modules with Maven + shell: bash + if: ${{ matrix.it_task == 'others'}} + run: | + mvn clean install -DskipTests + mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 0000000000000..30f29f16090ca --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,60 @@ +# This workflow will check if dependencies have changed (adding new dependencies or removing existing ones) + +name: Dependency Check + +on: + push: + branches: + - master + - 'rel/*' + - "rc/*" + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - "rc/*" + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + dependency-check: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: corretto + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Do the dependency check + shell: bash + run: mvn verify -Dmaven.test.skip=true -Dmdep.analyze.skip=true -P enable-sbom-check diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml index 4d323e9bb412d..5bba3d5d5ace5 100644 --- a/.github/workflows/multi-language-client.yml +++ b/.github/workflows/multi-language-client.yml @@ -3,17 +3,30 @@ on: push: branches: - master - - "rel/*" - paths-ignore: - - 'docs/**' - - 'site/**' + - "rc/*" + paths: + - 'pom.xml' + - 'iotdb-client/pom.xml' + - 'iotdb-client/client-py/**' + - 'iotdb-client/client-cpp/**' + - 'example/client-cpp-example/**' + - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' + - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' + - '.github/workflows/multi-language-client.yml' pull_request: branches: - master - - "rel/*" - paths-ignore: - - 'docs/**' - - 'site/**' + - "rc/*" + - 'force_ci/**' + paths: + - 'pom.xml' + - 'iotdb-client/pom.xml' + - 'iotdb-client/client-py/**' + - 'iotdb-client/client-cpp/**' + - 'example/client-cpp-example/**' + - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' + - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' + - '.github/workflows/multi-language-client.yml' # allow manually run the action: workflow_dispatch: @@ -32,35 +45,44 @@ jobs: fail-fast: false max-parallel: 15 matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-22.04, ubuntu-24.04, windows-2019, windows-2022, macos-latest] runs-on: ${{ matrix.os}} steps: - uses: actions/checkout@v4 - name: Install CPP Dependencies (Ubuntu) - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' shell: bash run: | sudo apt-get update sudo apt-get install libboost-all-dev - name: Install CPP Dependencies (Mac) - if: matrix.os == 'macos-latest' + # remove some xcode to release disk space + if: runner.os == 'macOS' shell: bash run: | brew install boost - brew install bison + sudo rm -rf /Applications/Xcode_14.3.1.app + sudo rm -rf /Applications/Xcode_15.0.1.app + sudo rm -rf /Applications/Xcode_15.1.app + sudo rm -rf /Applications/Xcode_15.2.app + sudo rm -rf /Applications/Xcode_15.3.app - name: Install CPP Dependencies (Windows) - if: matrix.os == 'windows-latest' + if: runner.os == 'Windows' run: | choco install winflexbison3 - choco install boost-msvc-14.3 + if ("${{ matrix.os }}" -eq "windows-2019") { + choco install boost-msvc-14.2 + } else { + choco install boost-msvc-14.3 + } $boost_path = (Get-ChildItem -Path 'C:\local\' -Filter 'boost_*').FullName echo $boost_path >> $env:GITHUB_PATH - name: Cache Maven packages uses: actions/cache@v4 with: path: ~/.m2 - key: client-${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- - name: Build IoTDB server shell: bash @@ -70,13 +92,20 @@ jobs: # Explicitly using mvnw here as the build requires maven 3.9 and the default installation is older # Explicitly using "install" instead of package in order to be sure we're using libs built on this machine # (was causing problems on windows, but could cause problem on linux, when updating the thrift module) - run: ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am + run: | + if [[ "$RUNNER_OS" == "Linux" ]]; then + ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Diotdb-tools-thrift.version=0.14.1.1-glibc223-SNAPSHOT + elif [[ "${{ matrix.os }}" == "windows-2019" ]]; then + ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Diotdb-tools-thrift.version=0.14.1.1-msvc142-SNAPSHOT -Dcmake.generator="Visual Studio 16 2019" + else + ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am + fi - name: Upload Artifact if: failure() uses: actions/upload-artifact@v4 with: name: cpp-IT-${{ runner.os }} - path: iotdb-client/client-cpp/target/build/test/Testing + path: distribution/target/apache-iotdb-*-all-bin/apache-iotdb-*-all-bin/logs retention-days: 1 go: @@ -98,9 +127,9 @@ jobs: - name: Integration test shell: bash run: | - cd iotdb-client - git clone https://github.com/apache/iotdb-client-go.git - cd iotdb-client-go + cd iotdb-client + git clone https://github.com/apache/iotdb-client-go.git + cd iotdb-client-go make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo python: @@ -108,10 +137,13 @@ jobs: fail-fast: false max-parallel: 15 matrix: - python: [ '3.6', '3.x' ] - runs-on: ${{ (matrix.python == '3.6' && 'ubuntu-20.04') || 'ubuntu-latest' }} + python: ['3.x'] + runs-on: ${{ 'ubuntu-latest' }} steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} - uses: actions/checkout@v4 - name: Cache Maven packages uses: actions/cache@v4 @@ -119,24 +151,26 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- + - name: Cache pip packages + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: ${{ runner.os }}-pip- - name: Build IoTDB server distribution zip and python client run: mvn -B clean install -pl distribution,iotdb-client/client-py -am -DskipTests - name: Build IoTDB server docker image run: | docker build . -f docker/src/main/Dockerfile-1c1d -t "iotdb:dev" docker images - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - name: Install IoTDB python client requirements run: pip3 install -r iotdb-client/client-py/requirements_dev.txt - name: Check code style if: ${{ matrix.python == '3.x'}} shell: bash run: black iotdb-client/client-py/ --check --diff - - name: Integration test + - name: Integration test and test make package shell: bash run: | cd iotdb-client/client-py/ && pytest . - - + ./release.sh diff --git a/.github/workflows/pipe-it-2cluster.yml b/.github/workflows/pipe-it-2cluster.yml deleted file mode 100644 index 676fae32d5139..0000000000000 --- a/.github/workflows/pipe-it-2cluster.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: Multi-Cluster IT - -on: - push: - branches: - - master - - 'rel/1.*' - paths-ignore: - - 'docs/**' - - 'site/**' - pull_request: - branches: - - master - - 'rel/1.*' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - -jobs: - auto-create-schema: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: liberica - java-version: ${{ matrix.java }} - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2AutoCreateSchema - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: cluster-log-auto-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - manual-create-schema: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] - cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: liberica - java-version: ${{ matrix.java }} - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2ManualCreateSchema - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: cluster-log-manual-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [ 17 ] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode ] - cluster2: [ ScalableSingleNodeMode ] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: liberica - java-version: ${{ matrix.java }} - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2Subscription - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: cluster-log-subscription-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml new file mode 100644 index 0000000000000..ba09fb4e27e31 --- /dev/null +++ b/.github/workflows/pipe-it.yml @@ -0,0 +1,935 @@ +name: Multi-Cluster IT + +on: + push: + branches: + - master + - 'rel/*' + - 'rc/*' + paths-ignore: + - 'docs/**' + - 'site/**' + - 'iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**' #queryengine + pull_request: + branches: + - master + - 'rel/*' + - 'rc/*' + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + - 'iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**' #queryengine + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + single: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] + os: [ ubuntu-latest ] + exclude: + - cluster1: LightWeightStandaloneMode + cluster2: LightWeightStandaloneMode + - cluster1: LightWeightStandaloneMode + cluster2: ScalableSingleNodeMode + - cluster1: ScalableSingleNodeMode + cluster2: LightWeightStandaloneMode + - cluster1: ScalableSingleNodeMode + cluster2: HighPerformanceMode + - cluster1: HighPerformanceMode + cluster2: LightWeightStandaloneMode + - cluster1: HighPerformanceMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusBatchMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusBatchMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusStreamMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusStreamMode + cluster2: HighPerformanceMode + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT1 \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-tree-auto-basic: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTreeAutoBasic \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-tree-auto-enhanced: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] + os: [ ubuntu-latest ] + exclude: + - cluster1: LightWeightStandaloneMode + cluster2: LightWeightStandaloneMode + - cluster1: LightWeightStandaloneMode + cluster2: ScalableSingleNodeMode + - cluster1: ScalableSingleNodeMode + cluster2: LightWeightStandaloneMode + - cluster1: ScalableSingleNodeMode + cluster2: HighPerformanceMode + - cluster1: HighPerformanceMode + cluster2: LightWeightStandaloneMode + - cluster1: HighPerformanceMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusBatchMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusBatchMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusStreamMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusStreamMode + cluster2: HighPerformanceMode + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTreeAutoEnhanced \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-tree-manual: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] + os: [ ubuntu-latest ] + exclude: + - cluster1: LightWeightStandaloneMode + cluster2: LightWeightStandaloneMode + - cluster1: LightWeightStandaloneMode + cluster2: ScalableSingleNodeMode + - cluster1: ScalableSingleNodeMode + cluster2: LightWeightStandaloneMode + - cluster1: ScalableSingleNodeMode + cluster2: HighPerformanceMode + - cluster1: HighPerformanceMode + cluster2: LightWeightStandaloneMode + - cluster1: HighPerformanceMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusBatchMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusBatchMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusStreamMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusStreamMode + cluster2: HighPerformanceMode + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTreeManual \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + subscription-tree-arch-verification: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [ ScalableSingleNodeMode ] + cluster2: [ ScalableSingleNodeMode ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2SubscriptionTreeArchVerification \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + subscription-table-arch-verification: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [ ScalableSingleNodeMode ] + cluster2: [ ScalableSingleNodeMode ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2SubscriptionTableArchVerification \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + subscription-tree-regression-consumer: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal + cluster1: [ ScalableSingleNodeMode ] + cluster2: [ ScalableSingleNodeMode ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + subscription-tree-regression-misc: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal + cluster1: [ ScalableSingleNodeMode ] + cluster2: [ ScalableSingleNodeMode ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-table-manual-basic: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTableManualBasic \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-table-manual-enhanced: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTableManualEnhanced \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + path: integration-test/target/cluster-logs + retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml index 1c5c4827401b9..638bf34a3c539 100644 --- a/.github/workflows/sonar-codecov.yml +++ b/.github/workflows/sonar-codecov.yml @@ -8,6 +8,7 @@ on: branches: - master - "rel/*" + - "rc/*" paths-ignore: - "docs/**" - 'site/**' @@ -16,6 +17,8 @@ on: - master - "rel/*" - "new_*" + - "rc/*" + - 'force_ci/**' paths-ignore: - "docs/**" - 'site/**' @@ -30,7 +33,7 @@ env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 MAVEN_ARGS: --batch-mode --no-transfer-progress PR_NUMBER: ${{ github.event.number }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: codecov: diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml new file mode 100644 index 0000000000000..b04101c6479b0 --- /dev/null +++ b/.github/workflows/table-cluster-it-1c1d.yml @@ -0,0 +1,89 @@ +name: Table Cluster IT - 1C1D + +on: + push: + branches: + - master + - 'rel/*' + - 'rc/*' + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - 'rc/*' + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + Simple: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + os: [ ubuntu-latest, windows-latest ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: 17 + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Adjust network dynamic TCP ports range + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + netsh int ipv4 set dynamicport tcp start=32768 num=32768 + netsh int ipv4 set dynamicport udp start=32768 num=32768 + netsh int ipv6 set dynamicport tcp start=32768 num=32768 + netsh int ipv6 set dynamicport udp start=32768 num=32768 + - name: Adjust Linux kernel somaxconn + if: ${{ runner.os == 'Linux' }} + shell: bash + run: sudo sysctl -w net.core.somaxconn=65535 +# - name: Adjust Mac kernel somaxconn +# if: ${{ runner.os == 'macOS' }} +# shell: bash +# run: sudo sysctl -w kern.ipc.somaxconn=65535 + - name: IT/UT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \ + -pl integration-test \ + -am -PTableSimpleIT + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} + path: integration-test/target/cluster-logs + retention-days: 1 + diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml new file mode 100644 index 0000000000000..a2603dd79138f --- /dev/null +++ b/.github/workflows/table-cluster-it-1c3d.yml @@ -0,0 +1,67 @@ +name: Table Cluster IT - 1C3D + +on: + push: + branches: + - master + - 'rel/*' + - 'rc/*' + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - 'rc/*' + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + Simple: + strategy: + fail-fast: false + max-parallel: 20 + matrix: + java: [ 17 ] + runs-on: [self-hosted, iotdb] +# group: self-hosted +# labels: iotdb + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: IT/UT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ + -pl integration-test \ + -am -PTableClusterIT + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: table-cluster-log-java${{ matrix.java }}-${{ runner.os }} + path: integration-test/target/cluster-logs + retention-days: 1 diff --git a/.github/workflows/todos-check.yml b/.github/workflows/todos-check.yml new file mode 100644 index 0000000000000..602edfb461201 --- /dev/null +++ b/.github/workflows/todos-check.yml @@ -0,0 +1,54 @@ +name: Check TODOs and FIXMEs in Changed Files + +on: + pull_request: + branches: + - master + - 'dev/*' + - 'rel/*' + - "rc/*" + - 'force_ci/**' + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +jobs: + todo-check: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check for TODOs and FIXMEs in changed files + run: | + # Fetch the target branch + git fetch origin $GITHUB_BASE_REF + + git switch -c check_branch + + # Get the diff of the changes + echo Get the diff of the changes + DIFF=$(git diff origin/$GITHUB_BASE_REF check_branch -- . ':(exclude).github/workflows/todos-check.yml') + + if [ -z "$DIFF" ]; then + echo "No changes detected." + exit 0 + fi + + + # Check the diff for TODOs + + # Check the diff for TODOs + echo Check the diff for TODOs + TODOsCOUNT=$(echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | wc -l) + if [ "$TODOsCOUNT" -eq 0 ]; then + echo "No TODOs or FIXMEs found in changed content."; + exit 0 + fi + + echo "TODO or FIXME found in the changes. Please resolve it before merging." + echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | tee -a output.log + exit 1 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 2379c66efe824..34fb0262f9952 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -8,7 +8,7 @@ on: branches: - master - 'rel/*' - - pipe-meta-sync + - "rc/*" paths-ignore: - 'docs/**' - 'site/**' @@ -16,7 +16,8 @@ on: branches: - master - 'rel/*' - - pipe-meta-sync + - "rc/*" + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' @@ -28,9 +29,9 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Xms2g -Xmx4g -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: unit-test: @@ -38,16 +39,9 @@ jobs: fail-fast: false max-parallel: 15 matrix: - java: [ 8, 17 ] + java: [ 17 ] os: [ ubuntu-latest, windows-latest ] it_task: [ 'others', 'datanode' ] - include: - - java: 17 - os: macos-latest - it_task: 'datanode' - - java: 17 - os: macos-latest - it_task: 'others' runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/vulnerability-check.yml b/.github/workflows/vulnerability-check.yml new file mode 100644 index 0000000000000..d85db02d1f6fa --- /dev/null +++ b/.github/workflows/vulnerability-check.yml @@ -0,0 +1,54 @@ +name: vulnerability-check +on: + schedule: + # Run at UTC 16:00 every week (CST 00:00 AM) + - cron: '0 16 * * 0' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + +jobs: + dependency-check: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: corretto + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Do the dependency-check:check + shell: bash + run: mvn org.owasp:dependency-check-maven:check + - name: Do the dependency-check:aggregate + shell: bash + run: mvn org.owasp:dependency-check-maven:aggregate + - name: Convert UTC to East Asia Standard Time and Extract Date + run: | + utc_time="${{ github.run_started_at }}" + target_time=$(TZ=Asia/Shanghai date -d "$utc_time" +"%Y-%m-%d") + echo "DATE_EAST_ASIA=$target_time" >> $GITHUB_ENV + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: vulnerability-check-result-${{ runner.os }}-${{ env.DATE_EAST_ASIA }} + path: target/dependency-check-report.html + retention-days: 15 diff --git a/.idea/icon.png b/.idea/icon.png new file mode 100644 index 0000000000000..493aca9320ef7 Binary files /dev/null and b/.idea/icon.png differ diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml index b505d1a366647..60509d3e8efef 100644 --- a/.mvn/develocity.xml +++ b/.mvn/develocity.xml @@ -20,8 +20,9 @@ --> + iotdb - https://ge.apache.org + https://develocity.apache.org #{isFalse(env['GITHUB_ACTIONS'])} diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index f3f1983375a6e..0836fc47d0100 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,11 +23,11 @@ com.gradle develocity-maven-extension - 1.21.3 + 1.22.2 com.gradle common-custom-user-data-maven-extension - 2.0 + 2.0.1 diff --git a/LICENSE b/LICENSE index 9a30cbd72803b..6e1d4e59d3d95 100644 --- a/LICENSE +++ b/LICENSE @@ -204,88 +204,44 @@ APACHE IOTDB SUBCOMPONENTS -------------------------------------------------------------------------------- -The following class is copied from maven-wrapper (https://github.com/takari/maven-wrapper), -which is under Apache License 2.0: - -mvnw files from https://github.com/apache/maven-wrapper Apache 2.0 - --------------------------------------------------------------------------------- - -The following class is modified from Apache commons-collections - -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Murmur128Hash.java -Relevant pr is: https://github.com/apache/commons-collections/pull/83/ - --------------------------------------------------------------------------------- - -The following files include code modified from Michael Burman's gorilla-tsc project. - -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntGorillaEncoder.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongGorillaEncoder.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV2.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntGorillaDecoder.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongGorillaDecoder.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java - -Copyright: 2016-2018 Michael Burman and/or other contributors -Project page: https://github.com/burmanm/gorilla-tsc -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -The following files include code modified from Panagiotis Liakos, Katia Papakonstantinopoulou and Yannis Kotidis chimp project. - -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionChimpDecoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntChimpDecoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongChimpDecoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionChimpDecoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionChimpEncoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntChimpEncoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongChimpEncoder.java -./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionChimpEncoder.java - -Copyright: 2022- Panagiotis Liakos, Katia Papakonstantinopoulou and Yannis Kotidis -Project page: https://github.com/panagiotisl/chimp -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - The following files include code modified from Apache HBase project. -./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/Procedure.java -./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/ProcedureExecutor.java -./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/StateMachineProcedure.java -./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/TimeoutExecutorThread.java -./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/StoppableThread.java +./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java +./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java +./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java +./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java +./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java Project page: https://github.com/apache/hbase -License: http://www.apache.org/licenses/LICENSE-2.0 +License: https://github.com/apache/hbase/blob/master/LICENSE.txt -------------------------------------------------------------------------------- -The following files include code modified from Eclipse Collections project. +The following files include code modified from Largest-Triangle downsampling algorithm implementations for Java8 project, +which is under Apache License 2.0: -./tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ByteArrayList.java +./library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Area.java +./library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Bucket.java +./library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/LTThreeBuckets.java +./library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/OnePassBucketizer.java +./library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/SlidingCollector.java +./library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Triangle.java -Copyright: 2021 Goldman Sachs -Project page: https://www.eclipse.org/collections -License: https://github.com/eclipse/eclipse-collections/blob/master/LICENSE-EDL-1.0.txt +Copyright: 2016 Guillermo Gutierrez Almazor +Project page: https://github.com/ggalmazor/lt_downsampling_java8 +License: https://github.com/ggalmazor/lt_downsampling_java8/blob/main/LICENSE -------------------------------------------------------------------------------- The following files include code modified from Micrometer project. -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmClassLoaderMetrics -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmCompileMetrics -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmGcMetrics -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmMemoryMetrics -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmThreadMetrics -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/logback/LogbackMetrics -./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/JvmUtils +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmClassLoaderMetrics.java +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmCompileMetrics.java +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmMemoryMetrics.java +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmThreadMetrics.java +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/JvmUtils.java +./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/logback/logback/LogbackMetrics.java Copyright: 2017 VMware Project page: https://github.com/micrometer-metrics/micrometer @@ -293,8 +249,7 @@ License: https://github.com/micrometer-metrics/micrometer/blob/main/LICENSE -------------------------------------------------------------------------------- -The following files include code modified from Trino project(https://github.com/trinodb/trino), -which is under Apache License 2.0: +The following files include code modified from Trino project. ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/QueryState.java ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/StateMachine.java @@ -302,5 +257,16 @@ which is under Apache License 2.0: ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceState.java ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachine.java ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java -./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/* +Trino is open source software licensed under the Apache License 2.0 and supported by the Trino Software Foundation. +Project page: https://github.com/trinodb/trino +License: https://github.com/trinodb/trino/blob/master/LICENSE + +-------------------------------------------------------------------------------- + +The following files include code modified from Apache Kafka project. + +./iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/PollTimer.java + +Project page: https://github.com/apache/kafka +License: https://github.com/apache/kafka/blob/trunk/LICENSE diff --git a/LICENSE-binary b/LICENSE-binary index 82b6baeea32f3..c088b00d7f3f5 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -215,96 +215,90 @@ following license. See licenses/ for text of these licenses. Apache Software Foundation License 2.0 -------------------------------------- -commons-cli:commons-cli:1.3.1 -commons-codec:commons-codec:1.13 +commons-cli:commons-cli:1.5.0 +commons-codec:commons-codec:1.16.1 org.apache.commons:commons-collections4:4.4 -commons-io:commons-io:2.5 -org.apache.commons:commons-lang3:3.8.1 -commons-lang:commons-lang:2.6 -com.nimbusds:content-type:2.0 -com.google.code.gson:gson:2.8.6 +commons-io:commons-io:2.14.0 +org.apache.commons:commons-lang3:3.13.0 +com.nimbusds:content-type:2.2 +com.google.code.gson:gson:2.10.1 com.google.guava.guava:32.1.2-jre -com.fasterxml.jackson.core:jackson-annotations:2.10.0 -com.fasterxml.jackson.core:jackson-core:2.10.0 -com.fasterxml.jackson.core:jackson-databind:2.10.0 -javax.inject:javax.inject:1 -net.jpountz.lz4:1.3.0 +com.fasterxml.jackson.core:jackson-annotations:2.15.4 +com.fasterxml.jackson.core:jackson-core:2.15.4 +com.fasterxml.jackson.core:jackson-databind:2.15.4 +jakarta.inject:jakarta.inject:2.6.1 +org.lz4:lz4-java:1.8.0 com.github.stephenc.jcip:jcip-annotations:1.0-1 -com.github.ben-manes.caffeine:caffeine:2.9.1 -org.eclipse.jetty:jetty-http:9.4.24.v20191120 -org.eclipse.jetty:jetty-io:9.4.24.v20191120 -org.eclipse.jetty:jetty-security:9.4.24.v20191120 -org.eclipse.jetty:jetty-server:9.4.24.v20191120 -org.eclipse.jetty:jetty-servlet:9.4.24.v20191120 -org.eclipse.jetty:jetty-util:9.4.24.v20191120 -org.eclipse.jetty:jetty-webapp:9.4.24.v20191120 -org.eclipse.jetty:jetty-xml:9.4.24.v20191120 -io.jsonwebtoken:jjwt-api:0.10.7 -io.jsonwebtoken:jjwt-impl:0.10.7 -io.jsonwebtoken:jjwt-jackson:0.10.7 -net.minidev:json-smart:2.3 +com.github.ben-manes.caffeine:caffeine:2.9.3 +org.eclipse.jetty:jetty-http:9.4.56.v20240826 +org.eclipse.jetty:jetty-io:9.4.56.v20240826 +org.eclipse.jetty:jetty-security:9.4.56.v20240826 +org.eclipse.jetty:jetty-server:9.4.56.v20240826 +org.eclipse.jetty:jetty-servlet:9.4.56.v20240826 +org.eclipse.jetty:jetty-util:9.4.56.v20240826 +io.jsonwebtoken:jjwt-api:0.11.5 +io.jsonwebtoken:jjwt-impl:0.11.5 +io.jsonwebtoken:jjwt-jackson:0.11.5 +net.minidev:json-smart:2.5.0 com.google.code.findbugs:jsr305:3.0.2 -com.nimbusds:lang-tag:1.4.4 +com.nimbusds:lang-tag:1.7 com.librato.metrics:librato-java:2.1.0 org.apache.thrift:libthrift:0.14.1 -io.dropwizard.metrics:metrics-core:3.2.6 -io.dropwizard.metrics:metrics-json:3.2.6 -io.dropwizard.metrics:metrics-jvm:3.2.6 +io.dropwizard.metrics:metrics-core:4.2.19 +io.dropwizard.metrics:metrics-jvm:3.2.2 com.librato.metrics:metrics-librato:5.1.0 -de.fraunhofer.iosb.io.moquette:moquette-broker:0.14.3 -io.netty:netty-buffer:4.1.53.Final -io.netty:netty-codec:4.1.53.Final -io.netty:netty-codec-http:4.1.53.Final -io.netty:netty-codec-mqtt:4.1.53.Final -io.netty:netty-common:4.1.53.Final -io.netty:netty-handler:4.1.53.Final -io.netty:netty-resolver:4.1.53.Final -io.netty:netty-transport:4.1.53.Final -io.netty:netty-transport-native-epoll:4.1.53.Final:linux-x86_64 -io.netty:netty-transport-native-unix-common:4.1.53.Final -com.nimbusds:nimbus-jose-jwt:8.14.1 -com.nimbusds:oauth2-oidc-sdk:8.3 -org.osgi:org.osgi.core:6.0.0 -org.osgi:osgi.cmpn:6.0.0 -org.ops4j.pax.jdbc:pax-jdbc-common:1.4.5 -org.xerial.snappy:snappy-java:1.1.8.4 -io.airlift.airline:0.8 -net.minidev:accessors-smart:1.2 +de.fraunhofer.iosb.io.moquette:moquette-broker:0.17 +io.netty:netty-buffer:4.1.110.Final +io.netty:netty-codec:4.1.110.Final +io.netty:netty-codec-http:4.1.110.Final +io.netty:netty-codec-mqtt:4.1.110.Final +io.netty:netty-common:4.1.110.Final +io.netty:netty-handler:4.1.110.Final +io.netty:netty-resolver:4.1.110.Final +io.netty:netty-transport:4.1.110.Final +io.netty:netty-transport-native-epoll:4.1.110.Final:linux-x86_64 +io.netty:netty-transport-native-unix-common:4.1.110.Final +com.nimbusds:nimbus-jose-jwt:9.37.3 +com.nimbusds:oauth2-oidc-sdk:10.15 +org.osgi:org.osgi.core:7.0.0 +org.osgi:osgi.cmpn:7.0.0 +org.ops4j.pax.jdbc:pax-jdbc-common:1.5.6 +org.xerial.snappy:snappy-java:1.1.10.5 +io.airlift.airline:0.9 +net.minidev:accessors-smart:2.5.0 BSD 3-Clause ------------ -org.antlr:antlr-runtime:4.8-1 -org.ow2.asm:asm:5.0.4 -org.jline:jline:3.21.0 +org.antlr:antlr-runtime:4.9.3 +org.ow2.asm:asm:9.3 +org.jline:jline:3.26.2 BSD 2-Clause ------------ -com.github.luben:zstd-jni:1.5.4-2 +com.github.luben:zstd-jni:1.5.6-3 MIT License ------------ -org.slf4j:slf4j-api -me.tongfei:progressbar:0.7.3 -com.bugsnag:bugsnag:3.6.1 -org.slf4j:jcl-over-slf4j:1.7.25 +org.slf4j:slf4j-api:2.0.9 +com.bugsnag:bugsnag:3.7.2 EPL 1.0 ------------ -com.h2database:h2-mvstore:1.4.199 -ch.qos.logback:logback-classic:1.2.10 -ch.qos.logback:logback-core:1.2.10 +com.h2database:h2-mvstore:2.1.212 +ch.qos.logback:logback-classic:1.3.14 +ch.qos.logback:logback-core:1.3.14 CDDL 1.1 ------------ -javax.annotation:javax.annotation-api:1.3.2 -javax.servlet:javax.servlet-api:3.1.0 +jakarta.annotation:jakarta.annotation-api:1.3.5 +jakarta.servlet:jakarta.servlet-api:4.0.4 javax.xml.bind:jaxb-api:2.4.0-b180725.0427 -org.glassfish.jaxb:jaxb-runtime:2.4.0-b180725.0644 +org.glassfish.jaxb:jaxb-runtime:2.3.6 Public Domain diff --git a/README.md b/README.md index 6c5a2fc68b37f..fa0f249a95e04 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Mainly on the ARM-based models: Building `Thrift` requires us to add two more dependencies to the picture. -This however is only needed when enabling the `compile-cpp` profile: +This however is only needed when enabling the `with-cpp` profile: brew install boost brew install bison @@ -293,7 +293,7 @@ After being built, the IoTDB cli is located at the folder "cli/target". ### Build Others -Use `-P compile-cpp` for compiling the cpp client. (For more details, read client-cpp's Readme file.) +Use `-P with-cpp` for compiling the cpp client. (For more details, read client-cpp's Readme file.) **NOTE: Directories "`thrift/target/generated-sources/thrift`", "`thrift-sync/target/generated-sources/thrift`", "`thrift-cluster/target/generated-sources/thrift`", "`thrift-influxdb/target/generated-sources/thrift`" @@ -511,7 +511,7 @@ see [Frequent Questions when Compiling the Source Code](https://iotdb.apache.org ### Wechat Group -* Add friend: `tietouqiao` or `liutaohua001`, and then we'll invite you to the group. +* Add friend: `apache_iotdb`, and then we'll invite you to the group. ### Slack diff --git a/README_ZH.md b/README_ZH.md index 8b56048fc96e9..89d180f52591c 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -35,7 +35,7 @@ [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://join.slack.com/t/apacheiotdb/shared_invite/zt-qvso1nj8-7715TpySZtZqmyG5qXQwpg) # 简介 -IoTDB (Internet of Things Database) 是一款时序数据库管理系统，可以为用户提供数据收集、存储和分析等服务。IoTDB由于其轻量级架构、高性能和高可用的特性，以及与 Hadoop 和 Spark 生态的无缝集成，满足了工业 IoT 领域中海量数据存储、高吞吐量数据写入和复杂数据查询分析的需求。 +IoTDB (Internet of Things Database) 是一款[时序数据库管理系统](https://www.timecho.com/archives/48)，可以为用户提供数据收集、存储和分析等服务。IoTDB由于其轻量级架构、高性能和高可用的特性，以及与 Hadoop 和 Spark 生态的无缝集成，满足了工业 IoT 领域中海量数据存储、高吞吐量数据写入和复杂数据查询分析的需求。 更多功能请见[时序数据库IoTDB：功能详解与行业应用](https://www.timecho.com/archives/shi-xu-shu-ju-ku-iotdb-gong-neng-xiang-jie-yu-xing-ye-ying-yong)。 IoTDB 依赖 [TsFile](https://github.com/apache/tsfile) 项目，它是一种专门用于时序数据管理的文件格式. TsFile 仓库的 `iotdb` 分支被用来为 IoTDB master 分支部署 SNAPSHOT 版本. @@ -172,7 +172,7 @@ git checkout rel/x.x ### 编译其他模块 -通过添加 `-P compile-cpp` 可以进行c++客户端API的编译。 +通过添加 `-P with-cpp` 可以进行c++客户端API的编译。 **注意："`thrift/target/generated-sources/thrift`"， "`thrift-sync/target/generated-sources/thrift`"，"`thrift-cluster/target/generated-sources/thrift`"，"`thrift-influxdb/target/generated-sources/thrift`" 和 "`antlr/target/generated-sources/antlr4`" 目录需要添加到源代码根中，以免在 IDE 中产生编译错误。** @@ -392,7 +392,7 @@ server 可以使用 "ctrl-C" 或者执行下面的脚本: ### Wechat Group -* 添加好友 `tietouqiao` 或者 `liutaohua001`，我们会邀请您进群 +* 添加好友 `apache_iotdb`，我们会邀请您进群 ### Slack diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 6cc03a0e67601..1a50ea804b0c3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -19,6 +19,469 @@ --> +# Apache IoTDB 2.0.2 + +## Features & Improvements + +- Data Query: Added management of table model UDFs, including user-defined scalar functions (UDSF) and user-defined aggregate functions (UDAF). +- Data Query: Table model now supports permission management, user management, and authorization for related operations. +- Data Query: Introduced new system tables and various O&M statements to optimize system management. +- System Management: The tree model and table model are now fully isolated at the database level. +- System Management: The built-in MQTT Service is now compatible with the table model. +- System Management: The CSharp client now supports the table model. +- System Management: The Go client now supports the table model. +- System Management: Added a C++ Session write interface for the table model. +- Data Synchronization: The table model now supports metadata synchronization and synchronization delete operations. +- Scripts and Tools: The import-data/export-data scripts now support the table model and local TsFile Load. +- ... + +## Bugs + +- Fixed the memory leak issue when writing data using SQL. +- Fixed the issue of duplicate timestamps appearing in table model queries. +- Fixed the issue of duplicate removal anomaly in table model aggregate queries with GROUP BY. +- Fixed the handling of Long.MIN_VALUE or Long.MAX_VALUE during write and merge processes. +- Fixed the issue of Long.MIN_VALUE timestamp causing time partition overflow and subsequent load failure. +- Fixed the issue of out-of-order data within a single TSFile on the destination data node during region migration in load operations. +- Fixed the issue where Explain Analyze caused the execution plan to fail to properly perform column pruning. +- Fixed the issue where the C# Session could not correctly fetch result sets when querying large amounts of data (exceeding fetch_size) on a cluster with more than one node. +- Fixed the inconsistency in reading JDK environment variables by ConfigNode and DataNode on Windows. +- Fixed the issue where the query distribution time statistics in Explain Analyze were larger than actual values, and changed the query distribution time monitoring from FI level to Query level. + ... + +# Apache IoTDB 2.0.1-beta + +## Features & Improvements + +- Table Model: IoTDB has introduced a new model named table model, and supports standard SQL query syntax, including SELECT, WHERE, JOIN, GROUP BY, ORDER BY, LIMIT clause and subQuery. +- Data Query: The table model supports a variety of functions and operators, including logical operators, mathematical functions, and the time-series specific function DIFF, etc. +- Data Query: The databases of the table model and tree model are invisible to each other, and users can choose the appropriate model based on their needs. +- Data Query: Users can control the loading of UDF, PipePlugin, Trigger, and AINode via URI with configuration items to load JAR packages. +- Storage Engine: The table model supports data ingestion through the Session interface, and the Session interface supports automatic metadata creation. +- Storage Engine: The Python client now supports four new data types: String, Blob, Date, and Timestamp. +- Storage Engine: The comparison rules for the priority of same-type merge tasks have been optimized. +- Data Synchronization: Support for specifying authentication information of the receiving end at the sending end. +- Stream Processing Module: TsFile Load now supports the table model. +- Stream Processing Module: Pipe now supports the table model. +- System Management: The Benchmark tool has been adapted to support the table model. +- System Management: The Benchmark tool now supports four new data types: String, Blob, Date, and Timestamp. +- System Management: The stability of DataNode scaling down has been enhanced. +- System Management: Users are now allowed to perform drop database operations in readonly mode. +- Scripts and Tools: The import-data/export-data scripts have been extended to support new data types (string, binary large objects, date, timestamp). +- Scripts and Tools: The import-data/export-data scripts have been iterated to support the import and export of three types of data: TsFile, CSV, and SQL. +- Ecosystem Integration: Support for Kubernetes Operator. + ... + +## Bugs + +- Fixed the issue where the query result set contained duplicate timestamps. +- Fixed the issue where deleted data could be queried again when triggered to merge after deletion. +- Fixed the issue where the target sequence in SELECT INTO containing backticks would result in writing the wrong sequence. +- Fixed the issue where an array out-of-bounds exception was thrown in the HAVING clause of the tree model due to a non-existent column name. +- Fixed the issue where MergeReader needed to consider memory allocation to avoid negative available memory during out-of-order and reverse queries. +- Fixed the issue where the CN in the cluster could not register large pipe plugins (greater than 100MB) and the parameters were not configurable. +- Fixed the issue of controlling the memory size of TimeIndex referenced by Pipe for TsFileResource. +- Fixed the issue where the Storage Engine - File Count - mods displayed negative values on the monitoring dashboard. +- Fixed the issue where the query result order was incorrect in the C# client. + +... + +# Apache IoTDB 1.3.4 + +## Features & Improvements + +- Data Query: Users can now control the loading of JAR packages via URI for UDF, PipePlugin, Trigger, and AINode through configuration items. +- Data Query: Added monitoring for TimeIndex cached during the merge process. +- System Management: Expanded UDF functions with the addition of the pattern_match function for pattern matching. +- System Management:The Python session SDK now includes a parameter for connection timeout. +- System Management:Introduced authorization for cluster management-related operations. +- System Management:ConfigNode/DataNode now supports scaling down using SQL. +- System Management:ConfigNode automatically cleans up partition information exceeding the TTL (cleans up every 2 hours). +- Data Synchronization: Supports specifying authorization information for the receiver on the sender's end. +- Ecosystem Integration: Supports Kubernetes Operator. +- Scripts and Tools: The import-data/export-data scripts have been expanded to support new data types (strings, large binary objects, dates, timestamps). +- Scripts and Tools:The import-data/export-data scripts have been iterated to support importing and exporting data in three formats: TsFile, CSV, and SQL. + ... + +## Bugs + +- Fixed the issue where an ArrayIndexOutOfBoundsException occurred when a column name did not exist in the HAVING clause of the tree model. +- Fixed the issue where the target sequence in SELECT INTO contained backticks, resulting in incorrect sequences being written. +- Fixed the issue where an empty iot-consensus file was generated after an abnormal power outage, causing the DataNode (dn) to fail to start. +- Fixed the issue where the storage engine reported an error during asynchronous recovery after manually deleting the resource file, leading to Pipe startup failure. +- Fixed the issue where data forwarded by external Pipe could not be synchronized between dual-lives. +- Fixed the issue where the C# Session could not correctly fetch result sets when querying large amounts of data (exceeding fetch_size) on a cluster with more than one node. +- Fixed the issue where the order of query results was incorrect in the C# client. +- Fixed the issue where duplicate timestamps were included in query result sets. +- Fixed the issue where query results were incorrect for single-device queries with sort+offset+limit+align by device. +- Fixed the issue where data synchronization failed when a sequence S1 of data type A was deleted and then a sequence S1 of data type B was written, and a TTL existed. +- Fixed the issue where MergeReader needed to consider memory allocation to avoid negative available memory during out-of-order and reverse queries. +- Fixed the inconsistency in how ConfigNode and DataNode read the JDK environment variables on Windows. +- ... + +# Apache IoTDB 1.3.3 + +## Features & Improvements + +- Storage Engine: Added new data types String, Blob, Date, and Timestamp. +- Storage Engine: Multi-level storage has added a rate-limiting mechanism. +- Storage Engine: New merge target file splitting feature, with additional configuration file parameters, and improved memory control performance of the merge module. +- Data Query: Filter performance optimization, enhancing the speed of aggregate queries and where condition queries. +- Data Query: New client query requests load balancing optimization. +- Data Query: New active metadata statistics query added. +- Data Query: Optimized memory control strategy during the query planning phase. +- Data Synchronization: The sender supports transferring files to a specified directory, and the receiver automatically loads them into IoTDB. +- Data Synchronization: The receiver has a new automatic conversion mechanism for data type requests. +- Data Synchronization: Enhanced observability on the receiver side, supporting ops/latency statistics for multiple internal interfaces, consolidated into a single pipeTransfer display. +- Data Loading: DataNode actively listens and loads TsFiles, with additional observability metrics. +- Stream Processing Module: New data subscription capability, supporting subscription to database data in the form of data points or tsfile files. +- Stream Processing Module: Alter Pipe supports the ability to alter the source. +- System Management: Optimized configuration files, with the original three configuration files merged into one, reducing user operational costs. +- System Management: Optimized restart recovery performance, reducing startup time. +- System Management: Internal addition of monitoring items such as device count, estimated remaining time for data synchronization, size of data to be synchronized, and synchronization speed. +- Scripts and Tools: The import-tsfile script is expanded to support running the script on a different server from the IoTDB server. +- Scripts and Tools: New metadata import and export scripts added. +- Scripts and Tools: New support for Kubernetes Helm added. +- AINode: AINode module added. + ... + +## Bugs + +- Fixed the issue of NullPointerException (NPE) when merging chunks with modifications and empty pages in the sequential space. +- Fixed the issue where the wrong parent file was used when reassigning the file position for skipped files during merge, leading to failure in creating hard links. +- Fixed the issue where the newly added four data types had null values written, and the TsFile handling of the STRING type was incorrect, causing a BufferUnderflowException: null. +- Fixed the issue in the high availability scenario where stopping the DataNode resulted in a PipeException: Failed to start consensus pipe. +- Fixed the issue in Stream mode where the first batch of written data points might require a flush to be synchronized. +- Fixed the compatibility issue with pipe plugin upgrades. +- Fixed the issue where the `ORDER BY` clause became ineffective when used in combination with `LIMIT` in the last query. + ... + +# Apache IoTDB 1.3.2 + +## Features & Improvements + +- Storage Module: Performance improvement in the insertRecords interface for writing +- Query Module: New Explain Analyze statement added (monitoring the time spent on each stage of a single SQL execution) +- Query Module: New UDAF (User-Defined Aggregate Function) framework added +- Query Module: New MaxBy/MinBy functions added, supporting the retrieval of maximum/minimum values along with the corresponding timestamps +- Query Module: Performance improvement in value filtering queries +- Data Synchronization: Path matching supports path pattern +- Data Synchronization: Supports metadata synchronization (including time series and related attributes, permissions, etc.) +- Stream Processing: Added Alter Pipe statement, supporting hot updates of plugins for Pipe tasks +- System Module: System data point count statistics now include statistics for data imported by loading TsFile +- Scripts and Tools: New local upgrade backup tool added (backing up original data through hard links) +- Scripts and Tools: New export-data/import-data scripts added, supporting data export in CSV, TsFile formats or SQL statements +- Scripts and Tools: Windows environment now supports distinguishing ConfigNode, DataNode, and Cli by window name + ... +- +## Bugs + +- Optimize the error message when a NullPointerException (NPE) occurs due to a timeout when dropping a database. +- Add logs for notifyLeaderReady, notifyLeaderChanged, and procedure worker. +- Add compatibility handling for existing erroneous data during file merging. +- Fix the deadlock issue caused by flushing empty files during querying. +- Fix the issue where Ratis becomes unresponsive during read, write, and delete operations. +- Fix the concurrent bug in load and merge operations. +- Fix the issue where the system's compression ratio is recorded as a negative number in the file for certain scenarios. +- Fix the ConcurrentModificationException issue during memory estimation for merge tasks. +- Fix potential deadlocks that may occur when writing, automatically creating, and deleting databases concurrently. + ... + +# Apache IoTDB 1.3.1 + +## Features & Improvements +- Add cluster script for one-click start-stop (start-all/stop-all.sh & start-all/stop-all.bat) +- Add script for one-click instance information collection (collect-info.sh & collect-info.bat) +- Add new statistical aggregators include stddev and variance +- Add repair tsfile data command +- Support setting timeout threshold for Fill clause. When time beyond the threshold, do not fill value. +- Simplify the time range specification for data synchronization, directly set start and end times +- Improved system observability (adding dispersion monitoring of cluster nodes, observability of distributed task scheduling framework) +- Optimized default log output strategy +- Enhance memory control for Load TsFile, covering the entire process +- Rest API (Version 2) adds column type return. +- Improve the process of query execution +- Session automatically fetch all available DataNodes + ... + +# Apache IoTDB 1.3.0 + +## Bugs + +- Fix issue with abnormal behavior when time precision is not in milliseconds during grouping by month. +- Fix issue with abnormal behavior when duration contains multiple units during grouping by month. +- Fix bug where limit and offset cannot be pushed down when there is an order by clause. +- Fix abnormal behavior in combination scenario of grouping by month + align by device + limit. +- Fix deserialization errors during IoT protocol synchronization. +- Fix concurrent exceptions in deleting timeseries. +- Fix issue where group by level in view sequences does not execute correctly. +- Fix potential issue of metadata creation failure when increasing election timeout + +## Features & Improvements + +- Optimize the permission module and support timeseries permission control +- Optimize heap and off-heap memory configuration in startup script +- Computed-type view timeseries support LAST queries +- Add pipe-related monitoring metrics +- Pipe rename 'Extractor' to 'Source' and 'Connector' to 'Sink' +- [IOTDB-6138] Support Negative timestamp +- [IOTDB-6193] Reject Node startup when loading configuration file failed +- [IOTDB-6194] Rename target_config_node_list to seed_config_node +- [IOTDB-6200] Change schema template to device template +- [IOTDB-6207] Add Write Point Metrics for load function +- [IOTDB-6208] Node error detection through broken thrift pipe +- [IOTDB-6217] When the number of time series reaches the upper limit, the prompt message should be changed to prioritize using device templates +- [IOTDB-6218] Rename storage_query_schema_consensus_free_memory_proportion to datanode_memory_proportion +- [IOTDB-6219] Fix the display problem of explain that the print result is not aligned +- [IOTDB-6220] Pipe: Add check logic to avoid self-transmission +- [IOTDB-6222] Optimize the performance of Python client +- [IOTDB-6230] Add HEAPDUMP configuration in datanode-env.sh +- [IOTDB-6231]SchemaCache supports precise eviction +- [IOTDB-6232] Adding SSL function to dn_rpc_port + +# Apache IoTDB 1.2.2 + +## Bugs + +- [IOTDB-6160] while using ` in target path, select into will throw error +- [IOTDB-6167] DataNode can't register to cluster when fetch system configuration throws NPE +- [IOTDB-6168] ConfigNode register retry logic does not worked +- [IOTDB-6171] NPE will be thrown while printing FI with debug on +- [IOTDB-6184] Merge Sort finishes one iterator too long +- [IOTDB-6191] Fix group by year not considering leap years +- [IOTDB-6226] Fix the problem of inaccurate GC monitor detection at the beginning and adjusting the alert threshold +- [IOTDB-6239] Show regions display error create time + +## Features & Improvements + +- [IOTDB-6029] Implementing flink-sql-iotdb-connector +- [IOTDB-6084] Pipe: support node-urls in connector-v1 +- [IOTDB-6103] Adding count_time aggregation feature +- [IOTDB-6112] Limit & Offset push down doesn't take effect while there exist time filter +- [IOTDB-6115] Limit & Offset push down doesn't take effect while there exist null value +- [IOTDB-6120] push down limit/offset in query with group by time +- [IOTDB-6129] ConfigNode restarts without relying on Seed-ConfigNode +- [IOTDB-6131] Iotdb rest service supports insertRecords function +- [IOTDB-6151] Move DataNode's system.properties to upper dir +- [IOTDB-6173] Change default encoder of INT32 and INT64 from RLE to TS_2DIFF +- Adjust the default thrift timeout parameter to 60s +- Accelerate the deletion execution + +## Bugs + +- [IOTDB-6064] Pipe: Fix deadlock in rolling back procedures concurrently +- [IOTDB-6081] Pipe: use HybridExtractor instead of LogExtractor when realtime mode is set to log to avoid OOM under heavy insertion load +- [IOTDB-6145] Pipe: can not release TsFile or WAL resource after pipe is dropped +- [IOTDB-6146] Pipe: can not transfer data after 1000+ pipes' creating and dropping +- [IOTDB-6082] Improve disk space metrics +- [IOTDB-6104] tmp directory won't be cleaned after udf query end +- [IOTDB-6119] Add ConfigNode leader service check +- [IOTDB-6125] Fix DataPartition allocation bug when insert big batch data +- [IOTDB-6127] Pipe: buffered events in processor stage can not be consumed by connector +- [IOTDB-6132] CrossSpaceCompaction: The estimated memory size is too large for cross space compaction task +- [IOTDB-6133] NullPointerException occurs in unsequence InnerSpaceCompactionTask +- [IOTDB-6148] Pipe: Fixed the bug that some uncommited progresses may be reported +- [IOTDB-6156] Fixed TConfiguration invalidly in Thrift AsyncServer For IoTConsensus +- [IOTDB-6164] Can create illegal path through rest api +- Fix datanode status is ReadOnly because the disk is full +- Fix DataPartition allocation bug when insert big batch +- Fix flush point statistics +- Fix SchemaFileSketchTool is not found +- Refactoring DeleteOutdatedFileTask in WalNode +- Add compression and encoding type check for FastCompactionPerformer +- Add lazy page reader for aligned page reader to avoid huge memory cost when reading rows of aligned timeseries +- Pipe: use PipeTaskCoordinatorLock instead of ReentrantLock for multi thread sync +- Pipe: fix pipe procedure stuck because of data node async request forever waiting for response +- Pipe: fix NPE when HybridProgressIndex.updateToMinimumIsAfterProgressIndex after system reboot (DR: SimpleConsensus) +- Pipe: fix pipe coordinator deadlock causing CN election timeout +- Pipe: Improve performance for 10000+ pipes +- RATIS-1873. Remove RetryCache assertion that doesn't hold + +# Apache IoTDB 1.2.1 + +## Features & Improvements + +- [IOTDB-5557] The metadata query results are inconsistent +- [IOTDB-5997] Improve efficiency of ConfigNode PartitionInfo loadSnapshot +- [IOTDB-6019] Fix concurrent update of last query +- [IOTDB-6036] The mods file is too large, causing Query very slow even OOM problem +- [IOTDB-6055] Enable auto restart of the pipes stopped by ConfigNode because of critical exception +- [IOTDB-6066] Add ConfigNode timeslot metric +- [IOTDB-6073] Add ClientManager metrics +- [IOTDB-6077] Add force stop +- [IOTDB-6079] Cluster computing resource balance +- [IOTDB-6082] Improve disk space metrics +- [IOTDB-6087] Implement stream interface of Mods read +- [IOTDB-6090] Add memory estimator on inner space compaction +- [IOTDB-6092] Factor mods files into memory estimates for cross-space compaction tasks +- [IOTDB-6093] Add multiple validation methods after compaction +- [IOTDB-6106] Fixed the timeout parameter not working in thrift asyncClient +- [IOTDB-6108] AlignedTVList memory calculation is imprecise +- +## Bugs + +- [IOTDB-5855] DataRegion leader Distribution is same as DataRegion Distribution +- [IOTDB-5860] Total Number of file is wrong +- [IOTDB-5996] Incorrect time display of show queries +- [IOTDB-6057] Resolve the compatibility from 1.1.x to 1.2.0 +- [IOTDB-6065] Considering LastCacheContainer in the memory estimation of SchemaCacheEntry +- [IOTDB-6074] Ignore error message when TagManager createSnapshot +- [IOTDB-6075] Pipe: File resource races when different tsfile load operations concurrently modify the same tsfile at receiver +- [IOTDB-6076] Add duplicate checking when upsert alias +- [IOTDB-6078] fix timeChunk default compressType +- [IOTDB-6089] Improve the lock behaviour of the pipe heartbeat +- [IOTDB-6091] Add compression and encoding type check for FastCompactionPerformer +- [IOTDB-6094] Load：Fix construct tsFileResource bug +- [IOTDB-6095] Tsfiles in sequence space may be overlap with each other due to LastFlushTime bug +- [IOTDB-6096] M4 will output zero while meeting null +- [IOTDB-6097] ipe subscription running with the pattern option may cause OOM +- [IOTDB-6098] Flush error when writing aligned timeseries +- [IOTDB-6100] Pipe: Fix running in hybrid mode will cause wal cannot be deleted & some pipe data lost due to wrong ProducerType of Disruptor +- [IOTDB-6105] Load: NPE when analyzing tsfile + +# Apache IoTDB 1.2.0 + +## New Feature + +* [IOTDB-5567] add SQL for querying seriesslotid and timeslotid +* [IOTDB-5631] Add a built-in aggregation functions named time_duration +* [IOTDB-5636] Add round as built-in scalar function +* [IOTDB-5637] Add substr as built-in scalar function +* [IOTDB-5638] Support case when syntax in IoTDB +* [IOTDB-5643] Add REPLACE as a built-in scalar function +* [IOTDB-5683] Support aggregation function Mode for query +* [IOTDB-5711] Python API should support connecting multiple nodes +* [IOTDB-5752] Python Client supports write redirection +* [IOTDB-5765] Support Order By Expression +* [IOTDB-5771] add SPRINTZ and RLBE encodor and LZMA2 compressor +* [IOTDB-5924] Add SessionPool deletion API +* [IOTDB-5950] Support Dynamic Schema Template +* [IOTDB-5951] Support show timeseries/device with specific string contained in path +* [IOTDB-5955] Support create timeseries using schema template in Session API + +## Improvements + +* [IOTDB-5630] Make function cast a built-in function +* [IOTDB-5689] Close Isink when ISourceHandle is closed +* [IOTDB-5715] Improve the performance of query order by time desc +* [IOTDB-5763] Optimize the memory estimate for INTO operations +* [IOTDB-5887] Optimize the construction performance of PathPatternTree without wildcards +* [IOTDB-5888] TTL logs didn' t consider timestamp precision +* [IOTDB-5896] Failed to execute delete statement +* [IOTDB-5908] Add more query metrics +* [IOTDB-5911] print-iotdb-data-dir tool cannot work +* [IOTDB-5914] Remove redundant debug log in Session +* [IOTDB-5919] show variables add a variable timestamp_precision +* [IOTDB-5926] Optimize metric implementation +* [IOTDB-5929] Enable DataPartition inherit policy +* [IOTDB-5943] Avoid rpc invoking for SimpleQueryTerminator when endpoint is local address +* [IOTDB-5944] Follower doesn' t need to update last cache when using IoT_consensus +* [IOTDB-5945] Add a cache to avoid initialize duplicated device id object in write process +* [IOTDB-5946] Optimize the implement of tablet in Go client +* [IOTDB-5949] Support show timeseries with datatype filter +* [IOTDB-5952] Support FIFO strategy in DataNodeSchemaCache +* [IOTDB-6022] The WAL piles up when multi-replica iotconsensus is written at high concurrency + + +## Bug Fixes + +* [IOTDB-5604] NPE when execute Agg + align by device query without assigned DataRegion +* [IOTDB-5619] group by tags query NPE +* [IOTDB-5644] Unexpected result when there are no select expressions after analyzed in query +* [IOTDB-5657] Limit does not take effect in last query +* [IOTDB-5700] UDF query did not clean temp file after the query is finished +* [IOTDB-5716] Wrong dependency when pipeline consumeOneByOneOperator +* [IOTDB-5717] Incorrect result when querying with limit push-downing & order by time desc +* [IOTDB-5722] Wrong default execution branch in PlanVisitor +* [IOTDB-5735] The result of adding the distinct function to the align by device is incorrect +* [IOTDB-5755] Fix the problem that 123w can not be used in Identifier +* [IOTDB-5756] NPE when where predicate is NotEqualExpression and one of subExpression is not exist +* [IOTDB-5757] Not Supported Exception when use like ' s3 || false' in where even Type of s3 is Boolean +* [IOTDB-5760] Query is blocked because of no memory +* [IOTDB-5764] Cannot specify alias successfully when the FROM clause contains multiple path suffixes +* [IOTDB-5769] Offset doesn' t take effect in some special case +* [IOTDB-5774] The syntax that path nodes start or end with a wildcard to fuzzy match is not supported +* [IOTDB-5784] Incorrect result when querying with offset push-down and time filter +* [IOTDB-5815] NPE when using UDF to query +* [IOTDB-5837] Exceptions for select into using placeholders +* [IOTDB-5851] Using limit clause in show devices query will throw NPE +* [IOTDB-5858] Metric doesn' t display the schemaCache hit ratio +* [IOTDB-5861] Last quey is incomplete +* [IOTDB-5889] TTL Cannot delete expired tsfiles +* [IOTDB-5897] NullPointerException In compaction +* [IOTDB-5905] Some aligned timeseries data point lost after flush +* [IOTDB-5934] Optimize cluster partition policy +* [IOTDB-5953] LastCache memory control param does not take effect +* [IOTDB-5963] Sometimes we may get out-of-order query result +* [IOTDB-6016] Release file num cost after cross compaction task + + +# Apache IoTDB 1.1.2 +## New Feature + +* [IOTDB-5919]show variables add a variable timestamp_precision +* Add Python SessionPool + +## Improvement/Bugfix + +* [IOTDB-5901] Load: load tsfile without data will throw NPE +* [IOTDB-5903] Fix cannot select any inner space compaction task when there is only unsequence data +* [IOTDB-5878] Allow ratis-client retry when gRPC IO Unavailable +* [IOTDB-5939] Correct Flusing Task Timeout Detect Thread +* [IOTDB-5905] Fix aligned timeseries data point lost after flushed in some scenario +* [IOTDB-5963] Make sure that TsBlock blocked on memory is added in queue before the next TsBlock returned by root operator +* [IOTDB-5819] Fix npe when booting net metrics +* [IOTDB-6023] Pipe: Fix load tsfile error while handling empty value chunk +* [IOTDB-5971] Fix potential QUOTE problem in iotdb reporter +* [IOTDB-5993] ConfigNode leader changing causes lacking some DataPartition allocation result in the response of getOrCreateDataPartition method +* [IOTDB-5910] Fix compaction scheduler thread pool is not shutdown when aborting compaction +* [IOTDB-6056] Pipe: Failed to load tsfile with empty pages (NPE occurs when loading) +* [IOTDB-5916]Fix exception when file is deleted during compaction selection +* [IOTDB-5896] Fix the NPE issue when taking snapshot in WAL combined with Aligned Binary +* [IOTDB-5929] Enable DataPartition inherit policy +* [IOTDB-5934] Optimize cluster partition policy +* [IOTDB-5926] Remove Useless Rater in Timer +* [IOTDB-6030] Improve efficiency of ConfigNode PartitionInfo takeSnapshot +* [IOTDB-5997] Improve efficiency of ConfigNode PartitionInfo loadSnapshot +* Fix potential deadlock when freeing memory in MemoryPool +* Release resource of FI after all drivers have been closed +* Set default degree of parallelism back to the num of CPU +* Make SequenceStrategy and MaxDiskUsableSpaceFirstStrategy are allowed in cluster mode +* Fix npe exception when invalid in metric module +* Fix CQ does not take effect in ns time_precision +* Fix storage engine memory config initialization +* Fix template related schema query +* add default charset setting in start-datanode.bat and print default charset when starting +* Fix TsfileResource error after delete device in sequence working memtable +* load TsFile bugs: Not checking whether the tsfile data loaded locally is in the same time partition during the loading process & LoadTsFilePieceNode error when loading tsfile with empty value chunks +* Fix alias query failure after restarting DataNode + +# Apache IoTDB 1.1.1 +## New Feature + +* [IOTDB-2569] ZSTD compression + +## Improvement/Bugfix + +* [IOTDB-5781] Change the default strategy to SequenceStrategy +* [IOTDB-5780] Let users know a node was successfully removed and data is recovered +* [IOTDB-5735] The result of adding the distinct function to the align by device is incorrect +* [IOTDB-5777] When writing data using non-root users, the permission authentication module takes too long +* [IOTDB-5835] Fix wal accumulation caused by datanode restart +* [IOTDB-5828] Optimize the implementation of some metric items in the metric module to prevent Prometheus pull timeouts +* [IOTDB-5813] ConfigNode restart error due to installSnapshot failed +* [IOTDB-5657] Limit does not take effect in last query +* [IOTDB-5717] Incorrect result when querying with limit push-downing & order by time desc +* [IOTDB-5722] Wrong default execution branch in PlanVisitor +* [IOTDB-5784] Incorrect result when querying with offset push-down and time filter +* [IOTDB-5815] NPE when using UDF to query +* [IOTDB-5829] Query with limit clause will cause other concurrent query break down +* [IOTDB-5824] show devices with * cannot display satisfied devices +* [IOTDB-5831] Drop database won't delete totally files in disk +* [IOTDB-5818] Cross_space compaction of Aligned timeseries is stucked +* [IOTDB-5859] Compaction error when using Version as first sort dimension +* [IOTDB-5869] Fix load overlap sequence TsFile + # Apache IoTDB 1.1.0 ## New Features diff --git a/code-coverage/pom.xml b/code-coverage/pom.xml index aa1bd56ff3f6c..e4b39363839cd 100644 --- a/code-coverage/pom.xml +++ b/code-coverage/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-code-coverage pom diff --git a/dependencies.json b/dependencies.json new file mode 100644 index 0000000000000..2e121a3144403 --- /dev/null +++ b/dependencies.json @@ -0,0 +1,168 @@ +{ + "dependencies": [ + "cglib:cglib", + "ch.qos.logback:logback-classic", + "ch.qos.logback:logback-core", + "ch.qos.reload4j:reload4j", + "com.digitalpetri.fsm:strict-machine", + "com.digitalpetri.netty:netty-channel-fsm", + "com.fasterxml.jackson.core:jackson-annotations", + "com.fasterxml.jackson.core:jackson-core", + "com.fasterxml.jackson.core:jackson-databind", + "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", + "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", + "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", + "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", + "com.fasterxml.jackson.module:jackson-module-jaxb-annotations", + "com.github.ben-manes.caffeine:caffeine", + "com.github.luben:zstd-jni", + "com.github.stephenc.jcip:jcip-annotations", + "com.github.wendykierp:JTransforms", + "com.google.code.findbugs:jsr305", + "com.google.code.gson:gson", + "com.google.errorprone:error_prone_annotations", + "com.google.guava:failureaccess", + "com.google.guava:guava", + "com.google.guava:listenablefuture", + "com.google.j2objc:j2objc-annotations", + "com.h2database:h2-mvstore", + "com.lmax:disruptor", + "com.nimbusds:content-type", + "com.nimbusds:lang-tag", + "com.nimbusds:nimbus-jose-jwt", + "com.nimbusds:oauth2-oidc-sdk", + "com.sun.istack:istack-commons-runtime", + "com.zaxxer:HikariCP", + "commons-cli:commons-cli", + "commons-codec:commons-codec", + "commons-io:commons-io", + "commons-logging:commons-logging", + "io.airlift:airline", + "io.airlift:concurrent", + "io.airlift:log", + "io.airlift:units", + "io.dropwizard.metrics:metrics-core", + "io.jsonwebtoken:jjwt-api", + "io.micrometer:micrometer-commons", + "io.micrometer:micrometer-core", + "io.micrometer:micrometer-observation", + "com.github.moquette-io.moquette:moquette-broker", + "io.netty:netty-buffer", + "io.netty:netty-codec", + "io.netty:netty-codec-dns", + "io.netty:netty-codec-http", + "io.netty:netty-codec-http2", + "io.netty:netty-codec-mqtt", + "io.netty:netty-codec-socks", + "io.netty:netty-common", + "io.netty:netty-handler", + "io.netty:netty-handler-proxy", + "io.netty:netty-resolver", + "io.netty:netty-resolver-dns", + "io.netty:netty-resolver-dns-classes-macos", + "io.netty:netty-resolver-dns-native-macos", + "io.netty:netty-tcnative-boringssl-static", + "io.netty:netty-tcnative-classes", + "io.netty:netty-transport", + "io.netty:netty-transport-classes-epoll", + "io.netty:netty-transport-native-epoll", + "io.netty:netty-transport-native-unix-common", + "io.netty:netty-transport-classes-kqueue", + "io.netty:netty-transport-native-kqueue", + "io.projectreactor:reactor-core", + "io.projectreactor.netty:reactor-netty-core", + "io.projectreactor.netty:reactor-netty-http", + "io.swagger:swagger-annotations", + "io.swagger:swagger-core", + "io.swagger:swagger-jaxrs", + "io.swagger:swagger-models", + "jakarta.activation:jakarta.activation-api", + "jakarta.annotation:jakarta.annotation-api", + "jakarta.servlet:jakarta.servlet-api", + "jakarta.validation:jakarta.validation-api", + "jakarta.ws.rs:jakarta.ws.rs-api", + "jakarta.xml.bind:jakarta.xml.bind-api", + "net.java.dev.jna:jna", + "net.java.dev.jna:jna-platform", + "net.minidev:accessors-smart", + "net.minidev:json-smart", + "org.antlr:antlr4-runtime", + "org.apache.commons:commons-collections4", + "org.apache.commons:commons-csv", + "org.apache.commons:commons-jexl3", + "org.apache.commons:commons-lang3", + "org.apache.commons:commons-math3", + "org.apache.commons:commons-pool2", + "org.apache.httpcomponents:httpclient", + "org.apache.httpcomponents:httpcore", + "org.apache.ratis:ratis-client", + "org.apache.ratis:ratis-common", + "org.apache.ratis:ratis-grpc", + "org.apache.ratis:ratis-metrics-api", + "org.apache.ratis:ratis-proto", + "org.apache.ratis:ratis-server", + "org.apache.ratis:ratis-server-api", + "org.apache.ratis:ratis-thirdparty-misc", + "org.apache.thrift:libthrift", + "org.apache.tsfile:common", + "org.apache.tsfile:tsfile", + "org.bouncycastle:bcpkix-jdk18on", + "org.bouncycastle:bcprov-jdk18on", + "org.bouncycastle:bcutil-jdk18on", + "org.checkerframework:checker-qual", + "org.eclipse.collections:eclipse-collections", + "org.eclipse.collections:eclipse-collections-api", + "org.eclipse.jetty:jetty-http", + "org.eclipse.jetty:jetty-io", + "org.eclipse.jetty:jetty-security", + "org.eclipse.jetty:jetty-server", + "org.eclipse.jetty:jetty-servlet", + "org.eclipse.jetty:jetty-util", + "org.eclipse.jetty:jetty-util-ajax", + "org.eclipse.milo:bsd-core", + "org.eclipse.milo:bsd-generator", + "org.eclipse.milo:sdk-client", + "org.eclipse.milo:sdk-core", + "org.eclipse.milo:sdk-server", + "org.eclipse.milo:stack-client", + "org.eclipse.milo:stack-core", + "org.eclipse.milo:stack-server", + "org.fusesource.hawtbuf:hawtbuf", + "org.fusesource.hawtdispatch:hawtdispatch", + "org.fusesource.hawtdispatch:hawtdispatch-transport", + "org.fusesource.mqtt-client:mqtt-client", + "org.glassfish.hk2:hk2-api", + "org.glassfish.hk2:hk2-locator", + "org.glassfish.hk2:hk2-utils", + "org.glassfish.hk2:osgi-resource-locator", + "org.glassfish.hk2.external:aopalliance-repackaged", + "org.glassfish.hk2.external:jakarta.inject", + "org.glassfish.jaxb:jaxb-runtime", + "org.glassfish.jaxb:txw2", + "org.glassfish.jersey.containers:jersey-container-servlet-core", + "org.glassfish.jersey.core:jersey-client", + "org.glassfish.jersey.core:jersey-common", + "org.glassfish.jersey.core:jersey-server", + "org.glassfish.jersey.inject:jersey-hk2", + "org.glassfish.jersey.media:jersey-media-multipart", + "org.hdrhistogram:HdrHistogram", + "org.java-websocket:Java-WebSocket", + "org.javassist:javassist", + "org.jline:jline", + "org.jvnet.mimepull:mimepull", + "org.latencyutils:LatencyUtils", + "org.lz4:lz4-java", + "org.ops4j.pax.jdbc:pax-jdbc-common", + "org.osgi:osgi.cmpn", + "org.osgi:osgi.core", + "org.ow2.asm:asm", + "org.reactivestreams:reactive-streams", + "org.reflections:reflections", + "org.slf4j:slf4j-api", + "org.slf4j:slf4j-reload4j", + "org.tukaani:xz", + "org.xerial.snappy:snappy-java", + "org.yaml:snakeyaml", + "pl.edu.icm:JLargeArrays" + ] +} diff --git a/distribution/pom.xml b/distribution/pom.xml index 1f112d7fc2840..83fa9c070853f 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-distribution pom @@ -33,25 +33,25 @@ org.apache.iotdb iotdb-server - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT zip org.apache.iotdb iotdb-cli - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT zip org.apache.iotdb iotdb-confignode - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT zip org.apache.iotdb library-udf - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT @@ -168,5 +168,63 @@ + + with-ainode + + + org.apache.iotdb + iotdb-ainode + 2.0.4-SNAPSHOT + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + all-bin + + single + + package + + + src/assembly/ainode.xml + + + + + + + + net.nicoulaj.maven.plugins + checksum-maven-plugin + + + sign-source-release + + files + + package + + + + + apache-iotdb-${project.version}-ainode-bin.zip + + + + + + + + + + diff --git a/distribution/src/assembly/ainode.xml b/distribution/src/assembly/ainode.xml new file mode 100644 index 0000000000000..46bf51930b7ac --- /dev/null +++ b/distribution/src/assembly/ainode.xml @@ -0,0 +1,52 @@ + + + + ainode-bin + + dir + zip + + apache-iotdb-${project.version}-ainode-bin + + + ${project.basedir}/../iotdb-core/ainode/target/apache-iotdb-ainode-${project.version}/apache-iotdb-ainode-${project.version}/conf + ${file.separator}/conf + + + ${project.basedir}/../iotdb-core/ainode/target/apache-iotdb-ainode-${project.version}/apache-iotdb-ainode-${project.version}/sbin + ${file.separator}/sbin + 0755 + + + ${project.basedir}/../iotdb-core/ainode/target/apache-iotdb-ainode-${project.version}/apache-iotdb-ainode-${project.version}/lib + ${file.separator}/lib + + + ${project.basedir}/../iotdb-core/ainode/target/apache-iotdb-ainode-${project.version}/apache-iotdb-ainode-${project.version}/tools + ${file.separator}/tools + 0755 + + + + common-files.xml + + diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index c6da93929293e..b77f71a32dcd9 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -59,43 +59,32 @@ ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/conf - sbin - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/sbin - 0755 - - - sbin - ${project.basedir}/../iotdb-core/confignode/src/assembly/resources/sbin - 0755 - - - sbin - ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/sbin - 0755 - - - tools - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/tools + conf + ${project.basedir}/../scripts/conf + + ainode-env.* + **/ainode-env.* + 0755 sbin - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/sbin + ${project.basedir}/../scripts/sbin + + *ainode.* + **/*ainode.* + 0755 tools - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/tools + ${project.basedir}/../scripts/tools + + *ainode.* + **/*ainode.* + 0755 - - - - - - - - @@ -104,13 +93,6 @@ 0755 - - - - - - - common-files.xml diff --git a/distribution/src/assembly/cli.xml b/distribution/src/assembly/cli.xml index 1c62df4735f57..6f4eed7f7c259 100644 --- a/distribution/src/assembly/cli.xml +++ b/distribution/src/assembly/cli.xml @@ -38,13 +38,26 @@ + ${project.basedir}/../scripts/sbin sbin - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/sbin + + *cli.* + **/*cli.* + **/*cli-table.* + 0755 + ${project.basedir}/../scripts/tools tools - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/tools + + *data.* + *schema.* + *tsfile.* + **/*data.* + **/*schema.* + **/*tsfile.* + 0755 diff --git a/distribution/src/assembly/confignode.xml b/distribution/src/assembly/confignode.xml index abe88fce388a5..23665f09a2e99 100644 --- a/distribution/src/assembly/confignode.xml +++ b/distribution/src/assembly/confignode.xml @@ -38,28 +38,41 @@ - ${project.basedir}/../iotdb-core/confignode/src/assembly/resources/sbin - sbin - 0755 + ${project.basedir}/../iotdb-core/confignode/src/assembly/resources/conf + conf - sbin - ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/sbin - 0755 + conf + ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/conf - ${project.basedir}/../iotdb-core/confignode/src/assembly/resources/conf + ${project.basedir}/../scripts/conf conf + + iotdb-common.* + confignode-env.* + **/confignode-env.* + + 0755 - conf - ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/conf + ${project.basedir}/../scripts/sbin + sbin + + *confignode.* + **/*confignode.* + + 0755 + + + ${project.basedir}/../scripts/tools + tools + + *confignode.* + **/*confignode.* + + 0755 - - - - - common-files.xml diff --git a/distribution/src/assembly/datanode.xml b/distribution/src/assembly/datanode.xml index 9075bea61e445..016059a903a4f 100644 --- a/distribution/src/assembly/datanode.xml +++ b/distribution/src/assembly/datanode.xml @@ -38,47 +38,42 @@ - - - - conf ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/conf - sbin - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/sbin - 0755 - - - sbin - ${project.basedir}/../iotdb-core/node-commons/src/assembly/resources/sbin - 0755 - - - tools - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/tools + ${project.basedir}/../scripts/conf + conf + + datanode-env.* + **/datanode-env.* + 0755 + ${project.basedir}/../scripts/sbin sbin - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/sbin + + *datanode.* + **/*datanode.* + *cli.* + **/*cli.* + **/*cli-table.* + 0755 + ${project.basedir}/../scripts/tools tools - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/tools + + **/*confignode.* + **/*all.* + **/*ainode.* + 0755 - - - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh - conf/datanode-env.sh - 0755 - - common-files.xml diff --git a/docker/src/main/DockerCompose/do-docker-build.sh b/docker/src/main/DockerCompose/do-docker-build.sh index 19037f3834c36..f69d038910717 100755 --- a/docker/src/main/DockerCompose/do-docker-build.sh +++ b/docker/src/main/DockerCompose/do-docker-build.sh @@ -106,11 +106,11 @@ function prepare_buildx(){ docker buildx create --name mybuilder --driver docker-container --bootstrap --use docker run --rm --privileged tonistiigi/binfmt:latest --install all fi - find ${current_path}/../ -name 'Dockerfile-1.0.0*' | xargs sed -i 's#FROM eclipse-temurin:17-jre-focal#FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre-focal#g' + find ${current_path}/../ -name 'Dockerfile-1.0.0*' | xargs sed -i 's#FROM eclipse-temurin:17-jre-noble#FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre-noble#g' else docker_build="docker build" ; docker_publish="" ; - find ${current_path}/../ -name 'Dockerfile-1.0.0*' | xargs sed -i 's#FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre-focal#FROM eclipse-temurin:17-jre-focal#g' + find ${current_path}/../ -name 'Dockerfile-1.0.0*' | xargs sed -i 's#FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre-noble#FROM eclipse-temurin:17-jre-noble#g' fi } function build_iotdb(){ diff --git a/docker/src/main/DockerCompose/docker-compose-cluster-1c2d.yml b/docker/src/main/DockerCompose/docker-compose-cluster-1c2d.yml index d7fd52ead2f7f..0cd0f8ac78b4c 100644 --- a/docker/src/main/DockerCompose/docker-compose-cluster-1c2d.yml +++ b/docker/src/main/DockerCompose/docker-compose-cluster-1c2d.yml @@ -27,6 +27,7 @@ services: - cn_internal_port=10710 - cn_consensus_port=10720 - cn_seed_config_node=iotdb-confignode:10710 + - CONFIGNODE_JMX_OPTS=-Xms1G -Xmx1G -XX:MaxDirectMemorySize=256M volumes: - ./data/confignode:/iotdb/data - ./logs/confignode:/iotdb/logs @@ -48,6 +49,7 @@ services: - dn_mpp_data_exchange_port=10740 - dn_schema_region_consensus_port=10750 - dn_data_region_consensus_port=10760 + - IOTDB_JMX_OPTS=-Xms4G -Xmx4G -XX:MaxDirectMemorySize=1G volumes: - ./data/datanode1:/iotdb/data/ - ./logs/datanode1:/iotdb/logs/ @@ -67,6 +69,7 @@ services: - dn_mpp_data_exchange_port=10740 - dn_schema_region_consensus_port=10750 - dn_data_region_consensus_port=10760 + - IOTDB_JMX_OPTS=-Xms4G -Xmx4G -XX:MaxDirectMemorySize=1G volumes: - ./data/datanode2:/iotdb/data/ - ./logs/datanode2:/iotdb/logs/ diff --git a/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml b/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml index ab88c54f25486..242505d9d93b5 100644 --- a/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml +++ b/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml @@ -29,6 +29,7 @@ services: - schema_replication_factor=3 - schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus - config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus + - CONFIGNODE_JMX_OPTS=-Xms1G -Xmx1G -XX:MaxDirectMemorySize=256M volumes: - /etc/hosts:/etc/hosts:ro - ./data/confignode:/iotdb/data @@ -48,6 +49,7 @@ services: - dn_data_region_consensus_port=10760 - data_replication_factor=3 - data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus + - IOTDB_JMX_OPTS=-Xms4G -Xmx4G -XX:MaxDirectMemorySize=1G volumes: - /etc/hosts:/etc/hosts:ro - ./data/datanode:/iotdb/data/ diff --git a/docker/src/main/DockerCompose/docker-compose-standalone.yml b/docker/src/main/DockerCompose/docker-compose-standalone.yml index 7193acfdcde3f..1c8edd94eca3f 100644 --- a/docker/src/main/DockerCompose/docker-compose-standalone.yml +++ b/docker/src/main/DockerCompose/docker-compose-standalone.yml @@ -36,6 +36,8 @@ services: - dn_schema_region_consensus_port=10750 - dn_data_region_consensus_port=10760 - dn_seed_config_node=iotdb-service:10710 + - IOTDB_JMX_OPTS=-Xms4G -Xmx4G -XX:MaxDirectMemorySize=1G + - CONFIGNODE_JMX_OPTS=-Xms1G -Xmx1G -XX:MaxDirectMemorySize=256M volumes: - ./data:/iotdb/data - ./logs:/iotdb/logs diff --git a/docker/src/main/DockerCompose/entrypoint.sh b/docker/src/main/DockerCompose/entrypoint.sh index 72beea010eeb2..68df9db78b77d 100755 --- a/docker/src/main/DockerCompose/entrypoint.sh +++ b/docker/src/main/DockerCompose/entrypoint.sh @@ -26,11 +26,8 @@ function on_stop(){ if [[ "$start_what" != "confignode" ]]; then echo "###### manually flush ######"; start-cli.sh -e "flush;" || true - stop-datanode.sh - echo "##### done ######"; - else - stop-confignode.sh; fi + stop-standalone.sh } trap 'on_stop' SIGTERM SIGKILL SIGQUIT diff --git a/docker/src/main/DockerCompose/replace-conf-from-env.sh b/docker/src/main/DockerCompose/replace-conf-from-env.sh index 887faf1bdcd63..00a48050d7e6b 100755 --- a/docker/src/main/DockerCompose/replace-conf-from-env.sh +++ b/docker/src/main/DockerCompose/replace-conf-from-env.sh @@ -30,17 +30,22 @@ function process_single(){ if [[ -n "${line}" ]]; then echo "update $key $filename" local line_no=$(echo $line|cut -d : -f1) - local content=$(echo $line|cut -d : -f2) - if [[ "${content:0:1}" != "#" ]]; then + local content=$(echo $line|cut -d : -f2) + if [[ "${content:0:1}" != "#" ]]; then sed -i "${line_no}d" ${filename} fi - sed -i "${line_no} i${key_value}" ${filename} + sed -i "${line_no}a${key_value}" ${filename} + else + echo "append $key $filename" + line_no=$(wc -l $filename|cut -d ' ' -f1) + sed -i "${line_no}a${key_value}" ${filename} fi } function replace_configs(){ for v in $(env); do - if [[ "${v}" =~ "=" && "${v}" =~ "_" && ! "${v}" =~ "JAVA_" ]]; then + key_name="${v%%=*}" + if [[ "${key_name}" == "${key_name,,}" && ! 2w$key_name =~ ^_ ]]; then # echo "###### $v ####" for f in ${target_files}; do process_single $v ${conf_path}/$f diff --git a/docker/src/main/Dockerfile-1.0.0-confignode b/docker/src/main/Dockerfile-1.0.0-confignode index 7fd7319b1e8ed..ed8d1d43fe043 100644 --- a/docker/src/main/Dockerfile-1.0.0-confignode +++ b/docker/src/main/Dockerfile-1.0.0-confignode @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -FROM eclipse-temurin:17-jre-focal +FROM eclipse-temurin:17-jre-noble ARG version=1.0.0 ARG target=apache-iotdb-${version}-confignode-bin diff --git a/docker/src/main/Dockerfile-1.0.0-datanode b/docker/src/main/Dockerfile-1.0.0-datanode index 32f3c0fe17f64..62baeb27eed24 100644 --- a/docker/src/main/Dockerfile-1.0.0-datanode +++ b/docker/src/main/Dockerfile-1.0.0-datanode @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -FROM eclipse-temurin:17-jre-focal +FROM eclipse-temurin:17-jre-noble ARG version=1.0.0 ARG target=apache-iotdb-${version}-datanode-bin diff --git a/docker/src/main/Dockerfile-1.0.0-standalone b/docker/src/main/Dockerfile-1.0.0-standalone index da3a0ce149dae..79f846514295b 100644 --- a/docker/src/main/Dockerfile-1.0.0-standalone +++ b/docker/src/main/Dockerfile-1.0.0-standalone @@ -18,7 +18,7 @@ # # syntax=docker/dockerfile:1 -FROM eclipse-temurin:17-jre-focal +FROM eclipse-temurin:17-jre-noble ARG version=1.0.0 ARG target=apache-iotdb-${version}-all-bin diff --git a/docker/src/main/Dockerfile-1c1d b/docker/src/main/Dockerfile-1c1d index 996072066b71b..9a5605c1f2bdb 100644 --- a/docker/src/main/Dockerfile-1c1d +++ b/docker/src/main/Dockerfile-1c1d @@ -19,7 +19,7 @@ # docker build context is the root path of the repository -FROM eclipse-temurin:17-jre-focal +FROM eclipse-temurin:17-jre-noble ADD distribution/target/apache-iotdb-*-all-bin.zip / ADD docker/src/main/DockerCompose/start-1c1d.sh / @@ -42,7 +42,7 @@ RUN apt update \ && apt purge --auto-remove -y \ && apt clean -y \ RUN dos2unix /iotdb/sbin/start-1c1d.sh -RUN dos2unix /iotdb/sbin/iotdb-common.sh +RUN dos2unix /iotdb/sbin/../conf/iotdb-common.sh RUN dos2unix /iotdb/sbin/start-confignode.sh RUN dos2unix /iotdb/sbin/../conf/confignode-env.sh RUN dos2unix /iotdb/sbin/stop-confignode.sh diff --git a/example/client-cpp-example/pom.xml b/example/client-cpp-example/pom.xml index 29229f7078136..5b5bd2bdb3c55 100644 --- a/example/client-cpp-example/pom.xml +++ b/example/client-cpp-example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT client-cpp-example IoTDB: Example: CPP Client @@ -60,6 +60,10 @@ ${project.basedir}/src/AlignedTimeseriesSessionExample.cpp ${project.build.directory}/AlignedTimeseriesSessionExample.cpp + + ${project.basedir}/src/TableModelSessionExample.cpp + ${project.build.directory}/TableModelSessionExample.cpp + ${project.basedir}/src/CMakeLists.txt ${project.build.directory}/CMakeLists.txt diff --git a/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp b/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp index 8f175cb7650c6..2cc04ab3b3946 100644 --- a/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp +++ b/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp @@ -168,9 +168,9 @@ void insertAlignedTablet() { int randVal1 = 123456; double randVal2 = 123456.1234; double randVal3 = 123456.1234; - tablet.addValue(0, row, &randVal1); - tablet.addValue(1, row, &randVal2); - tablet.addValue(2, row, &randVal3); + tablet.addValue(0, row, randVal1); + tablet.addValue(1, row, randVal2); + tablet.addValue(2, row, randVal3); if (tablet.rowSize == tablet.maxRowNumber) { session->insertTablet(tablet, true); tablet.reset(); @@ -212,23 +212,23 @@ void insertAlignedTablets() { int randVal11 = rand(); int randVal12 = rand(); int randVal13 = rand(); - tablet1.addValue(0, row1, &randVal11); - tablet2.addValue(0, row2, &randVal12); - tablet3.addValue(0, row3, &randVal13); + tablet1.addValue(0, row1, randVal11); + tablet2.addValue(0, row2, randVal12); + tablet3.addValue(0, row3, randVal13); double randVal21 = rand() / 99.9; double randVal22 = rand() / 99.9; double randVal23 = rand() / 99.9; - tablet1.addValue(1, row1, &randVal21); - tablet2.addValue(1, row2, &randVal22); - tablet3.addValue(1, row3, &randVal23); + tablet1.addValue(1, row1, randVal21); + tablet2.addValue(1, row2, randVal22); + tablet3.addValue(1, row3, randVal23); bool randVal31 = (bool)(rand() % 2); bool randVal32 = (bool)(rand() % 2); bool randVal33 = (bool)(rand() % 2); - tablet1.addValue(2, row1, &randVal31); - tablet2.addValue(2, row2, &randVal32); - tablet3.addValue(2, row3, &randVal33); + tablet1.addValue(2, row1, randVal31); + tablet2.addValue(2, row2, randVal32); + tablet3.addValue(2, row3, randVal33); if (tablet1.rowSize == tablet1.maxRowNumber) { session->insertAlignedTablets(tabletMap, true); @@ -267,11 +267,11 @@ void insertNullableTabletWithAlignedTimeseries() { int64_t randVal2 = rand(); bool randVal3 = (bool)(rand() % 2); if (i == 0) { - tablet.addValue(i, row, &randVal1); + tablet.addValue(i, row, randVal1); } else if (i == 1) { - tablet.addValue(i, row, &randVal2); + tablet.addValue(i, row, randVal2); } else { - tablet.addValue(i, row, &randVal3); + tablet.addValue(i, row, randVal3); } // mark null value if ((row % 3) == (unsigned int) i) { diff --git a/example/client-cpp-example/src/CMakeLists.txt b/example/client-cpp-example/src/CMakeLists.txt index 91d93193e38b9..1b902ceaa5792 100644 --- a/example/client-cpp-example/src/CMakeLists.txt +++ b/example/client-cpp-example/src/CMakeLists.txt @@ -38,11 +38,14 @@ LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/client/lib) ADD_EXECUTABLE(SessionExample SessionExample.cpp) ADD_EXECUTABLE(AlignedTimeseriesSessionExample AlignedTimeseriesSessionExample.cpp) +ADD_EXECUTABLE(TableModelSessionExample TableModelSessionExample.cpp) IF(MSVC) TARGET_LINK_LIBRARIES(SessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib") TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib") + TARGET_LINK_LIBRARIES(TableModelSessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib") ELSE() TARGET_LINK_LIBRARIES(SessionExample iotdb_session pthread) TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session pthread) + TARGET_LINK_LIBRARIES(TableModelSessionExample iotdb_session pthread) ENDIF() diff --git a/example/client-cpp-example/src/SessionExample.cpp b/example/client-cpp-example/src/SessionExample.cpp index c3f5602cb08a8..2803d18976cbf 100644 --- a/example/client-cpp-example/src/SessionExample.cpp +++ b/example/client-cpp-example/src/SessionExample.cpp @@ -159,13 +159,13 @@ void insertTablet() { tablet.timestamps[row] = time; bool randVal1 = rand() % 2; - tablet.addValue(0, row, &randVal1); + tablet.addValue(0, row, randVal1); int randVal2 = rand(); - tablet.addValue(1, row, &randVal2); + tablet.addValue(1, row, randVal2); float randVal3 = (float)(rand() / 99.9); - tablet.addValue(2, row, &randVal3); + tablet.addValue(2, row, randVal3); if (tablet.rowSize == tablet.maxRowNumber) { session->insertTablet(tablet, true); @@ -236,22 +236,22 @@ void insertTablets() { tablet2.timestamps[row2] = time; int64_t randVal11 = rand(); - tablet1.addValue(0, row1, &randVal11); + tablet1.addValue(0, row1, randVal11); double randVal12 = rand() / 99.9; - tablet1.addValue(1, row1, &randVal12); + tablet1.addValue(1, row1, randVal12); string randVal13 = "string" + to_string(rand()); - tablet1.addValue(2, row1, &randVal13); + tablet1.addValue(2, row1, randVal13); int64_t randVal21 = rand(); - tablet2.addValue(0, row2, &randVal21); + tablet2.addValue(0, row2, randVal21); double randVal22 = rand() / 99.9; - tablet2.addValue(1, row2, &randVal22); + tablet2.addValue(1, row2, randVal22); string randVal23 = "string" + to_string(rand()); - tablet2.addValue(2, row2, &randVal23); + tablet2.addValue(2, row2, randVal23); if (tablet1.rowSize == tablet1.maxRowNumber) { session->insertTablets(tabletMap, true); @@ -292,7 +292,7 @@ void insertTabletWithNullValues() { tablet.timestamps[row] = time; for (int i = 0; i < 3; i++) { int64_t randVal = rand(); - tablet.addValue(i, row, &randVal); + tablet.addValue(i, row, randVal); // mark null value if (row % 3 == (unsigned int) i) { tablet.bitMaps[i].mark(row); diff --git a/example/client-cpp-example/src/TableModelSessionExample.cpp b/example/client-cpp-example/src/TableModelSessionExample.cpp new file mode 100644 index 0000000000000..4e4f9274342a6 --- /dev/null +++ b/example/client-cpp-example/src/TableModelSessionExample.cpp @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TableSession.h" +#include "TableSessionBuilder.h" + +using namespace std; + +TableSession *session; + +void insertRelationalTablet() { + + vector> schemaList { + make_pair("region_id", TSDataType::TEXT), + make_pair("plant_id", TSDataType::TEXT), + make_pair("device_id", TSDataType::TEXT), + make_pair("model", TSDataType::TEXT), + make_pair("temperature", TSDataType::FLOAT), + make_pair("humidity", TSDataType::DOUBLE) + }; + + vector columnTypes = { + ColumnCategory::TAG, + ColumnCategory::TAG, + ColumnCategory::TAG, + ColumnCategory::ATTRIBUTE, + ColumnCategory::FIELD, + ColumnCategory::FIELD + }; + + Tablet tablet("table1", schemaList, columnTypes, 100); + + for (int row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.timestamps[rowIndex] = row; + tablet.addValue("region_id", rowIndex, "1"); + tablet.addValue("plant_id", rowIndex, "5"); + tablet.addValue("device_id", rowIndex, "3"); + tablet.addValue("model", rowIndex, "A"); + tablet.addValue("temperature", rowIndex, 37.6F); + tablet.addValue("humidity", rowIndex, 111.1); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insert(tablet); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session->insert(tablet); + tablet.reset(); + } +} + +void Output(unique_ptr &dataSet) { + for (const string &name: dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; +} + +void OutputWithType(unique_ptr &dataSet) { + for (const string &name: dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + for (const string &type: dataSet->getColumnTypeList()) { + cout << type << " "; + } + cout << endl; + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; +} + +int main() { + try { + session = (new TableSessionBuilder()) + ->host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->build(); + + + cout << "[Create Database db1,db2]\n" << endl; + try { + session->executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + session->executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db2"); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Use db1 as database]\n" << endl; + try { + session->executeNonQueryStatement("USE db1"); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Create Table table1,table2]\n" << endl; + try { + session->executeNonQueryStatement("create table db1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + session->executeNonQueryStatement("create table db2.table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD) with (TTL=6600000)"); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Show Tables]\n" << endl; + try { + unique_ptr dataSet = session->executeQueryStatement("SHOW TABLES"); + Output(dataSet); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Show tables from specific database]\n" << endl; + try { + unique_ptr dataSet = session->executeQueryStatement("SHOW TABLES FROM db1"); + Output(dataSet); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[InsertTablet]\n" << endl; + try { + insertRelationalTablet(); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Query Table Data]\n" << endl; + try { + unique_ptr dataSet = session->executeQueryStatement("SELECT * FROM table1" + " where region_id = '1' and plant_id in ('3', '5') and device_id = '3'"); + OutputWithType(dataSet); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + session->close(); + + // specify database in constructor + session = (new TableSessionBuilder()) + ->host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->database("db1") + ->build(); + + cout << "[Show tables from current database(db1)]\n" << endl; + try { + unique_ptr dataSet = session->executeQueryStatement("SHOW TABLES"); + Output(dataSet); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Change database to db2]\n" << endl; + try { + session->executeNonQueryStatement("USE db2"); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Show tables from current database(db2)]\n" << endl; + try { + unique_ptr dataSet = session->executeQueryStatement("SHOW TABLES"); + Output(dataSet); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "[Drop Database db1,db2]\n" << endl; + try { + session->executeNonQueryStatement("DROP DATABASE db1"); + session->executeNonQueryStatement("DROP DATABASE db2"); + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + + cout << "session close\n" << endl; + session->close(); + + delete session; + + cout << "finished!\n" << endl; + } catch (IoTDBConnectionException &e) { + cout << e.what() << endl; + } catch (IoTDBException &e) { + cout << e.what() << endl; + } + return 0; +} \ No newline at end of file diff --git a/example/jdbc/pom.xml b/example/jdbc/pom.xml index 2eb25e0633e68..4ed8777aa2072 100644 --- a/example/jdbc/pom.xml +++ b/example/jdbc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT jdbc-example IoTDB: Example: JDBC diff --git a/example/jdbc/src/main/java/org/apache/iotdb/TableModelJDBCExample.java b/example/jdbc/src/main/java/org/apache/iotdb/TableModelJDBCExample.java new file mode 100644 index 0000000000000..49619b5639cf0 --- /dev/null +++ b/example/jdbc/src/main/java/org/apache/iotdb/TableModelJDBCExample.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.jdbc.IoTDBSQLException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +public class TableModelJDBCExample { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableModelJDBCExample.class); + + public static void main(String[] args) throws ClassNotFoundException, SQLException { + Class.forName("org.apache.iotdb.jdbc.IoTDBDriver"); + + // don't specify database in url + try (Connection connection = + DriverManager.getConnection( + "jdbc:iotdb://127.0.0.1:6667?sql_dialect=table", "root", "root"); + Statement statement = connection.createStatement()) { + + statement.execute("CREATE DATABASE test1"); + statement.execute("CREATE DATABASE test2"); + + statement.execute("use test2"); + + // or use full qualified table name + statement.execute( + "create table test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD) with (TTL=6600000)"); + + // show tables from current database + try (ResultSet resultSet = statement.executeQuery("SHOW TABLES")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + System.out.println(metaData.getColumnCount()); + while (resultSet.next()) { + System.out.println(resultSet.getString(1) + ", " + resultSet.getInt(2)); + } + } + + // show tables by specifying another database + // using SHOW tables FROM + try (ResultSet resultSet = statement.executeQuery("SHOW TABLES FROM test1")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + System.out.println(metaData.getColumnCount()); + while (resultSet.next()) { + System.out.println(resultSet.getString(1) + ", " + resultSet.getInt(2)); + } + } + + } catch (IoTDBSQLException e) { + LOGGER.error("IoTDB Jdbc example error", e); + } + + // specify database in url + try (Connection connection = + DriverManager.getConnection( + "jdbc:iotdb://127.0.0.1:6667/test1?sql_dialect=table", "root", "root"); + Statement statement = connection.createStatement()) { + // show tables from current database test1 + try (ResultSet resultSet = statement.executeQuery("SHOW TABLES")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + System.out.println(metaData.getColumnCount()); + while (resultSet.next()) { + System.out.println(resultSet.getString(1) + ", " + resultSet.getInt(2)); + } + } + + // change database to test2 + statement.execute("use test2"); + + try (ResultSet resultSet = statement.executeQuery("SHOW TABLES")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + System.out.println(metaData.getColumnCount()); + while (resultSet.next()) { + System.out.println(resultSet.getString(1) + ", " + resultSet.getInt(2)); + } + } + } + } +} diff --git a/example/mqtt-customize/pom.xml b/example/mqtt-customize/pom.xml index b67be1f441318..95d5ded1e208c 100644 --- a/example/mqtt-customize/pom.xml +++ b/example/mqtt-customize/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT customize-mqtt-example IoTDB: Example: Customized MQTT diff --git a/example/mqtt-customize/src/main/java/org/apache/iotdb/mqtt/server/CustomizedJsonPayloadFormatter.java b/example/mqtt-customize/src/main/java/org/apache/iotdb/mqtt/server/CustomizedJsonPayloadFormatter.java index 5b708f1bcd57d..2c56b2336601e 100644 --- a/example/mqtt-customize/src/main/java/org/apache/iotdb/mqtt/server/CustomizedJsonPayloadFormatter.java +++ b/example/mqtt-customize/src/main/java/org/apache/iotdb/mqtt/server/CustomizedJsonPayloadFormatter.java @@ -21,8 +21,10 @@ import org.apache.iotdb.db.protocol.mqtt.Message; import org.apache.iotdb.db.protocol.mqtt.PayloadFormatter; +import org.apache.iotdb.db.protocol.mqtt.TreeMessage; import io.netty.buffer.ByteBuf; +import org.apache.commons.lang3.NotImplementedException; import java.util.ArrayList; import java.util.Arrays; @@ -32,7 +34,7 @@ public class CustomizedJsonPayloadFormatter implements PayloadFormatter { @Override - public List format(ByteBuf payload) { + public List format(String topic, ByteBuf payload) { // Suppose the payload is a json format if (payload == null) { return Collections.emptyList(); @@ -43,7 +45,7 @@ public List format(ByteBuf payload) { // this is just an example, so we just generate some Messages directly for (int i = 0; i < 2; i++) { long ts = i; - Message message = new Message(); + TreeMessage message = new TreeMessage(); message.setDevice("d" + i); message.setTimestamp(ts); message.setMeasurements(Arrays.asList("s1", "s2")); @@ -53,9 +55,20 @@ public List format(ByteBuf payload) { return ret; } + @Override + @Deprecated + public List format(ByteBuf payload) { + throw new NotImplementedException(); + } + @Override public String getName() { // set the value of mqtt_payload_formatter in iotdb-common.properties as the following string: return "CustomizedJson"; } + + @Override + public String getType() { + return PayloadFormatter.TREE_TYPE; + } } diff --git a/example/mqtt/pom.xml b/example/mqtt/pom.xml index 62619735c0fc5..abeda271d8b25 100644 --- a/example/mqtt/pom.xml +++ b/example/mqtt/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT mqtt-example IoTDB: Example: MQTT diff --git a/example/mqtt/src/main/java/org/apache/iotdb/mqtt/MQTTClient.java b/example/mqtt/src/main/java/org/apache/iotdb/mqtt/MQTTClient.java index e0efc47393780..84815a85a4838 100644 --- a/example/mqtt/src/main/java/org/apache/iotdb/mqtt/MQTTClient.java +++ b/example/mqtt/src/main/java/org/apache/iotdb/mqtt/MQTTClient.java @@ -25,6 +25,9 @@ import java.util.Random; public class MQTTClient { + + private static final String DATABASE = "myMqttTest"; + public static void main(String[] args) throws Exception { MQTT mqtt = new MQTT(); mqtt.setHost("127.0.0.1", 1883); @@ -35,7 +38,14 @@ public static void main(String[] args) throws Exception { BlockingConnection connection = mqtt.blockingConnection(); connection.connect(); + // the config mqttPayloadFormatter must be tree-json + // jsonPayloadFormatter(connection); + // the config mqttPayloadFormatter must be table-line + linePayloadFormatter(connection); + connection.disconnect(); + } + private static void jsonPayloadFormatter(BlockingConnection connection) throws Exception { Random random = new Random(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < 10; i++) { @@ -58,7 +68,34 @@ public static void main(String[] args) throws Exception { sb.insert(0, "["); sb.replace(sb.lastIndexOf(","), sb.length(), "]"); connection.publish("root.sg.d1.s1", sb.toString().getBytes(), QoS.AT_LEAST_ONCE, false); + } - connection.disconnect(); + // The database must be created in advance + private static void linePayloadFormatter(BlockingConnection connection) throws Exception { + + String payload = + "test1,tag1=t1,tag2=t2 attr3=a5,attr4=a4 field1=\"fieldValue1\",field2=1i,field3=1u 1"; + connection.publish(DATABASE + "/myTopic", payload.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(10); + + payload = "test1,tag1=t1,tag2=t2 field4=2,field5=2i32,field6=2f 2"; + connection.publish(DATABASE, payload.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(10); + + payload = + "test1,tag1=t1,tag2=t2 field7=t,field8=T,field9=true 3 \n " + + "test1,tag1=t1,tag2=t2 field7=f,field8=F,field9=FALSE 4"; + connection.publish(DATABASE + "/myTopic", payload.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(10); + + payload = + "test1,tag1=t1,tag2=t2 attr1=a1,attr2=a2 field1=\"fieldValue1\",field2=1i,field3=1u 4 \n " + + "test1,tag1=t1,tag2=t2 field4=2,field5=2i32,field6=2f 5"; + connection.publish(DATABASE + "/myTopic", payload.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(10); + + payload = "# It's a remark\n " + "test1,tag1=t1,tag2=t2 field4=2,field5=2i32,field6=2f 6"; + connection.publish(DATABASE + "/myTopic", payload.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(10); } } diff --git a/example/pipe-count-point-processor/pom.xml b/example/pipe-count-point-processor/pom.xml index 9b486cd08bd54..64f869518b249 100644 --- a/example/pipe-count-point-processor/pom.xml +++ b/example/pipe-count-point-processor/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT pipe-count-point-processor-example IoTDB: Example: Pipe: Count Point Processor @@ -58,7 +58,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.2.2 diff --git a/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java b/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java index 1e13b27b3a3eb..53cc4f02a9656 100644 --- a/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java +++ b/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.pipe.api.PipeProcessor; +import org.apache.iotdb.pipe.api.annotation.TreeModel; import org.apache.iotdb.pipe.api.collector.EventCollector; import org.apache.iotdb.pipe.api.customizer.configuration.PipeProcessorRuntimeConfiguration; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; @@ -37,6 +38,7 @@ import java.util.Collections; import java.util.concurrent.atomic.AtomicLong; +@TreeModel public class CountPointProcessor implements PipeProcessor { private static final String AGGREGATE_SERIES_KEY = "aggregate-series"; private static final AtomicLong writePointCount = new AtomicLong(0); @@ -59,7 +61,7 @@ public void customize( public void process( final TabletInsertionEvent tabletInsertionEvent, final EventCollector eventCollector) { tabletInsertionEvent.processTablet( - (tablet, rowCollector) -> writePointCount.addAndGet(tablet.rowSize)); + (tablet, rowCollector) -> writePointCount.addAndGet(tablet.getRowSize())); } @Override @@ -67,15 +69,15 @@ public void process(final Event event, final EventCollector eventCollector) thro if (event instanceof PipeHeartbeatEvent) { final Tablet tablet = new Tablet( - aggregateSeries.getDevice(), + aggregateSeries.getIDeviceID().toString(), Collections.singletonList( new MeasurementSchema(aggregateSeries.getMeasurement(), TSDataType.INT64)), 1); - tablet.rowSize = 1; tablet.addTimestamp(0, System.currentTimeMillis()); tablet.addValue(aggregateSeries.getMeasurement(), 0, writePointCount.get()); eventCollector.collect( - new PipeRawTabletInsertionEvent(tablet, false, null, 0, null, null, false)); + new PipeRawTabletInsertionEvent( + false, null, null, null, tablet, false, null, 0, null, null, false)); } } diff --git a/example/pipe-opc-ua-sink/pom.xml b/example/pipe-opc-ua-sink/pom.xml index 37107b08de5e4..9c8f9d47f427f 100644 --- a/example/pipe-opc-ua-sink/pom.xml +++ b/example/pipe-opc-ua-sink/pom.xml @@ -23,7 +23,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT 4.0.0 pipe-opc-ua-sink-example diff --git a/example/pom.xml b/example/pom.xml index 548f7004d0d44..4e170a9d15692 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-examples pom diff --git a/example/rest-java-example/pom.xml b/example/rest-java-example/pom.xml index 970fe626dd9e5..9976d79737e20 100644 --- a/example/rest-java-example/pom.xml +++ b/example/rest-java-example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT rest-java-example IoTDB: Example: Java Rest diff --git a/example/rest-java-example/src/main/java/org/apache/iotdb/TableHttpExample.java b/example/rest-java-example/src/main/java/org/apache/iotdb/TableHttpExample.java new file mode 100644 index 0000000000000..ba419eddd7fff --- /dev/null +++ b/example/rest-java-example/src/main/java/org/apache/iotdb/TableHttpExample.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class TableHttpExample { + + private static final String UTF8 = "utf-8"; + + private String getAuthorization(String username, String password) { + return Base64.getEncoder() + .encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)); + } + + public static void main(String[] args) { + TableHttpExample httpExample = new TableHttpExample(); + httpExample.ping(); + httpExample.createDatabase(); + httpExample.createTable(); + httpExample.nonQuery(); + httpExample.insertTablet(); + httpExample.query(); + } + + public void ping() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + HttpGet httpGet = new HttpGet("http://127.0.0.1:18080/ping"); + CloseableHttpResponse response = null; + try { + response = httpClient.execute(httpGet); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + String result = JsonParser.parseString(message).getAsJsonObject().toString(); + System.out.println(result); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The ping rest api failed"); + } finally { + try { + httpClient.close(); + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Http Client close error"); + } + } + } + + private HttpPost getHttpPost(String url) { + HttpPost httpPost = new HttpPost(url); + httpPost.addHeader("Content-type", "application/json; charset=utf-8"); + httpPost.setHeader("Accept", "application/json"); + String authorization = getAuthorization("root", "root"); + httpPost.setHeader("Authorization", authorization); + return httpPost; + } + + public void insertTablet() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:18080/rest/table/v1/insertTablet"); + String json = + "{\"database\":\"test\",\"column_catogories\":[\"TAG\",\"ATTRIBUTE\",\"FIELD\"],\"timestamps\":[1635232143960,1635232153960,1635232163960,1635232173960,1635232183960],\"column_names\":[\"id1\",\"t1\",\"s1\"],\"data_types\":[\"STRING\",\"STRING\",\"FLOAT\"],\"values\":[[\"a11\",\"true\",11333],[\"a11\",\"false\",22333],[\"a13\",\"false1\",23333],[\"a14\",\"false2\",24],[\"a15\",\"false3\",25]],\"table\":\"sg211\"}"; + httpPost.setEntity(new StringEntity(json, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + String result = JsonParser.parseString(message).getAsJsonObject().toString(); + System.out.println(result); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The insertTablet rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void nonQuery() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:18080/rest/table/v1/nonQuery"); + String sql = + "{\"database\":\"test\",\"sql\":\"INSERT INTO sg211(time, id1, s1) values(100, 'd1', 0)\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The non query rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void createDatabase() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:18080/rest/table/v1/nonQuery"); + String sql = "{\"database\":\"\",\"sql\":\"create database if not exists test\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The non query rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void createTable() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:18080/rest/table/v1/nonQuery"); + String sql = + "{\"database\":\"test\",\"sql\":\"create table sg211 (id1 string TAG,t1 STRING ATTRIBUTE, s1 FLOAT FIELD)\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("create table failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void query() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:18080/rest/table/v1/query"); + String sql = "{\"database\":\"test\",\"sql\":\"select * from sg211\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The query rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } +} diff --git a/example/rest-java-example/src/main/java/org/apache/iotdb/TableHttpsExample.java b/example/rest-java-example/src/main/java/org/apache/iotdb/TableHttpsExample.java new file mode 100644 index 0000000000000..c6d8d4709e44e --- /dev/null +++ b/example/rest-java-example/src/main/java/org/apache/iotdb/TableHttpsExample.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class TableHttpsExample { + + private static final String UTF8 = "utf-8"; + + private String getAuthorization(String username, String password) { + return Base64.getEncoder() + .encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)); + } + + public static void main(String[] args) { + TableHttpsExample httpExample = new TableHttpsExample(); + httpExample.ping(); + httpExample.createDatabase(); + httpExample.createTable(); + httpExample.nonQuery(); + httpExample.insertTablet(); + httpExample.query(); + } + + public void ping() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + HttpGet httpGet = new HttpGet("https://127.0.0.1:18080/ping"); + CloseableHttpResponse response = null; + try { + response = httpClient.execute(httpGet); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + String result = JsonParser.parseString(message).getAsJsonObject().toString(); + System.out.println(result); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The ping rest api failed"); + } finally { + try { + httpClient.close(); + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Http Client close error"); + } + } + } + + private HttpPost getHttpPost(String url) { + HttpPost httpPost = new HttpPost(url); + httpPost.addHeader("Content-type", "application/json; charset=utf-8"); + httpPost.setHeader("Accept", "application/json"); + String authorization = getAuthorization("root", "root"); + httpPost.setHeader("Authorization", authorization); + return httpPost; + } + + public void insertTablet() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("https://127.0.0.1:18080/rest/table/v1/insertTablet"); + String json = + "{\"database\":\"test\",\"column_catogories\":[\"TAG\",\"ATTRIBUTE\",\"FIELD\"],\"timestamps\":[1635232143960,1635232153960,1635232163960,1635232173960,1635232183960],\"column_names\":[\"id1\",\"t1\",\"s1\"],\"data_types\":[\"STRING\",\"STRING\",\"FLOAT\"],\"values\":[[\"a11\",\"true\",11333],[\"a11\",\"false\",22333],[\"a13\",\"false1\",23333],[\"a14\",\"false2\",24],[\"a15\",\"false3\",25]],\"table\":\"sg211\"}"; + httpPost.setEntity(new StringEntity(json, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + String result = JsonParser.parseString(message).getAsJsonObject().toString(); + System.out.println(result); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The insertTablet rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void nonQuery() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("https://127.0.0.1:18080/rest/table/v1/nonQuery"); + String sql = + "{\"database\":\"test\",\"sql\":\"INSERT INTO sg211(time, id1, s1) values(100, 'd1', 0)\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The non query rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void createDatabase() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("https://127.0.0.1:18080/rest/table/v1/nonQuery"); + String sql = "{\"database\":\"\",\"sql\":\"create database if not exists test\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The non query rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void createTable() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("https://127.0.0.1:18080/rest/table/v1/nonQuery"); + String sql = + "{\"database\":\"test\",\"sql\":\"create table sg211 (id1 string TAG,t1 STRING ATTRIBUTE, s1 FLOAT FIELD)\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("create table failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } + + public void query() { + CloseableHttpClient httpClient = SSLClient.getInstance().getHttpClient(); + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("https://127.0.0.1:18080/rest/table/v1/query"); + String sql = "{\"database\":\"test\",\"sql\":\"select * from sg211\"}"; + httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, UTF8); + System.out.println(JsonParser.parseString(message).getAsJsonObject().toString()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println("The query rest api failed"); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + System.out.println("Response close error"); + } + } + } +} diff --git a/example/schema/pom.xml b/example/schema/pom.xml index 54246f9c8064a..28c5adb4ca6ec 100644 --- a/example/schema/pom.xml +++ b/example/schema/pom.xml @@ -24,7 +24,7 @@ iotdb-examples org.apache.iotdb - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT schema-example IoTDB: Example: Schema diff --git a/example/session/pom.xml b/example/session/pom.xml index 818cebd1ccc0a..4fd2c9998fc3d 100644 --- a/example/session/pom.xml +++ b/example/session/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT client-example IoTDB: Example: Session Client diff --git a/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java b/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java index 1bded669794a6..a9d3150e23b1e 100644 --- a/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java @@ -30,8 +30,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.IOException; @@ -300,7 +300,7 @@ private static void insertTabletWithAlignedTimeseriesMethod1() throws IoTDBConnectionException, StatementExecutionException { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); @@ -308,20 +308,21 @@ private static void insertTabletWithAlignedTimeseriesMethod1() long timestamp = 1; for (long row = 1; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue( - schemaList.get(0).getMeasurementId(), rowIndex, new SecureRandom().nextLong()); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, new SecureRandom().nextInt()); + schemaList.get(0).getMeasurementName(), rowIndex, new SecureRandom().nextLong()); + tablet.addValue( + schemaList.get(1).getMeasurementName(), rowIndex, new SecureRandom().nextInt()); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertAlignedTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertAlignedTablet(tablet); tablet.reset(); } @@ -334,31 +335,27 @@ private static void insertTabletWithAlignedTimeseriesMethod2() throws IoTDBConnectionException, StatementExecutionException { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); Tablet tablet = new Tablet(ROOT_SG1_D1_VECTOR2, schemaList); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; for (long time = 100; time < 200; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); - long[] sensor1 = (long[]) values[0]; - sensor1[row] = new SecureRandom().nextLong(); + tablet.addValue(row, 0, new SecureRandom().nextLong()); - int[] sensor2 = (int[]) values[1]; - sensor2[row] = new SecureRandom().nextInt(); + tablet.addValue(row, 1, new SecureRandom().nextInt()); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertAlignedTablet(tablet, true); tablet.reset(); } } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertAlignedTablet(tablet, true); tablet.reset(); } @@ -370,42 +367,29 @@ private static void insertNullableTabletWithAlignedTimeseries() throws IoTDBConnectionException, StatementExecutionException { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); Tablet tablet = new Tablet(ROOT_SG1_D1_VECTOR3, schemaList); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - // Use the bitMap to mark the null value point - BitMap[] bitMaps = new BitMap[values.length]; - tablet.bitMaps = bitMaps; - - bitMaps[1] = new BitMap(tablet.getMaxRowNumber()); for (long time = 200; time < 300; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - - long[] sensor1 = (long[]) values[0]; - sensor1[row] = new SecureRandom().nextLong(); + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); - int[] sensor2 = (int[]) values[1]; - sensor2[row] = new SecureRandom().nextInt(); + tablet.addValue(row, 0, new SecureRandom().nextLong()); - // mark this point as null value - if (time % 5 == 0) { - bitMaps[1].mark(row); + if (time % 5 != 0) { + tablet.addValue(row, 1, new SecureRandom().nextInt()); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertAlignedTablet(tablet, true); tablet.reset(); - bitMaps[1].reset(); } } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertAlignedTablet(tablet, true); tablet.reset(); } @@ -542,15 +526,15 @@ private static void insertAlignedRecordsOfOneDevice() private static void insertTabletsWithAlignedTimeseries() throws IoTDBConnectionException, StatementExecutionException { - List schemaList1 = new ArrayList<>(); + List schemaList1 = new ArrayList<>(); schemaList1.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList1.add(new MeasurementSchema("s2", TSDataType.INT64)); - List schemaList2 = new ArrayList<>(); + List schemaList2 = new ArrayList<>(); schemaList2.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList2.add(new MeasurementSchema("s2", TSDataType.INT64)); - List schemaList3 = new ArrayList<>(); + List schemaList3 = new ArrayList<>(); schemaList3.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList3.add(new MeasurementSchema("s2", TSDataType.INT64)); @@ -566,19 +550,19 @@ private static void insertTabletsWithAlignedTimeseries() // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 2; i++) { long value = new SecureRandom().nextLong(); - tablet1.addValue(schemaList1.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList2.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList3.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList1.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList2.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList3.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { session.insertAlignedTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -587,46 +571,7 @@ private static void insertTabletsWithAlignedTimeseries() timestamp++; } - if (tablet1.rowSize != 0) { - session.insertAlignedTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - - // Method 2 to add tablet data - long[] timestamps1 = tablet1.timestamps; - Object[] values1 = tablet1.values; - long[] timestamps2 = tablet2.timestamps; - Object[] values2 = tablet2.values; - long[] timestamps3 = tablet3.timestamps; - Object[] values3 = tablet3.values; - - for (long time = 0; time < 100; time++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; - timestamps1[row1] = time; - timestamps2[row2] = time; - timestamps3[row3] = time; - for (int i = 0; i < 2; i++) { - long[] sensor1 = (long[]) values1[i]; - sensor1[row1] = i; - long[] sensor2 = (long[]) values2[i]; - sensor2[row2] = i; - long[] sensor3 = (long[]) values3[i]; - sensor3[row3] = i; - } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { - session.insertAlignedTablets(tabletMap, true); - - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - } - - if (tablet1.rowSize != 0) { + if (tablet1.getRowSize() != 0) { session.insertAlignedTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); diff --git a/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java b/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java index 9abecb0d50719..fb130cce40a64 100644 --- a/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java +++ b/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java @@ -19,6 +19,7 @@ package org.apache.iotdb; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.isession.SessionDataSet.DataIterator; import org.apache.iotdb.isession.pool.SessionDataSetWrapper; @@ -28,6 +29,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,62 +132,64 @@ public Void call() { DataIterator dataIter = dataSet.iterator(); List columnNameList = dataIter.getColumnNameList(); List columnTypeList = dataIter.getColumnTypeList(); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); for (int j = 1; j < columnNameList.size(); j++) { - PartialPath currentPath = new PartialPath(columnNameList.get(j)); + PartialPath currentPath = new MeasurementPath(columnNameList.get(j)); schemaList.add( new MeasurementSchema( currentPath.getMeasurement(), TSDataType.valueOf(columnTypeList.get(j)))); } tablet = new Tablet(device, schemaList, 300000); while (dataIter.next()) { - int row = tablet.rowSize++; - tablet.timestamps[row] = dataIter.getLong(1); + int row = tablet.getRowSize(); + tablet.addTimestamp(row, dataIter.getLong(1)); for (int j = 0; j < schemaList.size(); ++j) { if (dataIter.isNull(j + 2)) { - tablet.addValue(schemaList.get(j).getMeasurementId(), row, null); + tablet.addValue(schemaList.get(j).getMeasurementName(), row, null); continue; } switch (schemaList.get(j).getType()) { case BOOLEAN: tablet.addValue( - schemaList.get(j).getMeasurementId(), row, dataIter.getBoolean(j + 2)); + schemaList.get(j).getMeasurementName(), row, dataIter.getBoolean(j + 2)); break; case INT32: - tablet.addValue(schemaList.get(j).getMeasurementId(), row, dataIter.getInt(j + 2)); + tablet.addValue( + schemaList.get(j).getMeasurementName(), row, dataIter.getInt(j + 2)); break; case INT64: case TIMESTAMP: - tablet.addValue(schemaList.get(j).getMeasurementId(), row, dataIter.getLong(j + 2)); + tablet.addValue( + schemaList.get(j).getMeasurementName(), row, dataIter.getLong(j + 2)); break; case FLOAT: tablet.addValue( - schemaList.get(j).getMeasurementId(), row, dataIter.getFloat(j + 2)); + schemaList.get(j).getMeasurementName(), row, dataIter.getFloat(j + 2)); break; case DOUBLE: tablet.addValue( - schemaList.get(j).getMeasurementId(), row, dataIter.getDouble(j + 2)); + schemaList.get(j).getMeasurementName(), row, dataIter.getDouble(j + 2)); break; case TEXT: case STRING: tablet.addValue( - schemaList.get(j).getMeasurementId(), row, dataIter.getString(j + 2)); + schemaList.get(j).getMeasurementName(), row, dataIter.getString(j + 2)); break; case DATE: case BLOB: tablet.addValue( - schemaList.get(j).getMeasurementId(), row, dataIter.getObject(j + 2)); + schemaList.get(j).getMeasurementName(), row, dataIter.getObject(j + 2)); break; default: LOGGER.info("Migration of this type of data is not supported"); } } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { writerPool.insertTablet(tablet, true); tablet.reset(); } } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { writerPool.insertTablet(tablet); tablet.reset(); } diff --git a/example/session/src/main/java/org/apache/iotdb/HybridTimeseriesSessionExample.java b/example/session/src/main/java/org/apache/iotdb/HybridTimeseriesSessionExample.java index 2a1f5d8a75aa9..34f529d257dbb 100644 --- a/example/session/src/main/java/org/apache/iotdb/HybridTimeseriesSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/HybridTimeseriesSessionExample.java @@ -26,6 +26,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +80,7 @@ private static void insertTabletWithAlignedTimeseriesMethod(int minTime, int max throws IoTDBConnectionException, StatementExecutionException { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); @@ -87,19 +88,19 @@ private static void insertTabletWithAlignedTimeseriesMethod(int minTime, int max long timestamp = minTime; for (long row = minTime; row < maxTime; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, row * 10 + 1L); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (int) (row * 10 + 2)); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, row * 10 + 1L); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (int) (row * 10 + 2)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertAlignedTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertAlignedTablet(tablet); tablet.reset(); } diff --git a/example/session/src/main/java/org/apache/iotdb/SessionConcurrentExample.java b/example/session/src/main/java/org/apache/iotdb/SessionConcurrentExample.java index 1aa3a962da8c5..40d3ed12e8094 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionConcurrentExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionConcurrentExample.java @@ -29,6 +29,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,52 +144,29 @@ private static void insertTablet(Session session, String deviceId) */ // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); Tablet tablet = new Tablet(deviceId, schemaList, 100); - // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = random.nextLong(); - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - - // Method 2 to add tablet data - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = i; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java index 0daec0885bf7f..e894c08406f2b 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -35,8 +35,8 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.IOException; @@ -393,53 +393,30 @@ private static void insertTablet() throws IoTDBConnectionException, StatementExe */ // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100); - // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = random.nextLong(); - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - - // Method 2 to add tablet data - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = i; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -457,77 +434,38 @@ private static void insertTabletWithNullValues() */ // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100); - // Method 1 to add tablet data insertTablet1(schemaList, tablet); - - // Method 2 to add tablet data - insertTablet2(schemaList, tablet); } - private static void insertTablet1(List schemaList, Tablet tablet) + private static void insertTablet1(List schemaList, Tablet tablet) throws IoTDBConnectionException, StatementExecutionException { tablet.initBitMaps(); long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = random.nextLong(); - // mark null value - if (row % 3 == s) { - tablet.bitMaps[s].mark((int) row); + if (row % 3 != s) { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - } - - private static void insertTablet2(List schemaList, Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - BitMap[] bitMaps = new BitMap[schemaList.size()]; - for (int s = 0; s < 3; s++) { - bitMaps[s] = new BitMap(tablet.getMaxRowNumber()); - } - tablet.bitMaps = bitMaps; - - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - // mark null value - if (row % 3 == i) { - bitMaps[i].mark(row); - } - sensor[row] = i; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -536,7 +474,7 @@ private static void insertTablet2(List schemaList, Tablet tab private static void insertTablets() throws IoTDBConnectionException, StatementExecutionException { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); @@ -550,22 +488,21 @@ private static void insertTablets() throws IoTDBConnectionException, StatementEx tabletMap.put("root.sg1.d2", tablet2); tabletMap.put("root.sg1.d3", tablet3); - // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 3; i++) { long value = random.nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { session.insertTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -574,46 +511,7 @@ private static void insertTablets() throws IoTDBConnectionException, StatementEx timestamp++; } - if (tablet1.rowSize != 0) { - session.insertTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - - // Method 2 to add tablet data - long[] timestamps1 = tablet1.timestamps; - Object[] values1 = tablet1.values; - long[] timestamps2 = tablet2.timestamps; - Object[] values2 = tablet2.values; - long[] timestamps3 = tablet3.timestamps; - Object[] values3 = tablet3.values; - - for (long time = 0; time < 100; time++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; - timestamps1[row1] = time; - timestamps2[row2] = time; - timestamps3[row3] = time; - for (int i = 0; i < 3; i++) { - long[] sensor1 = (long[]) values1[i]; - sensor1[row1] = i; - long[] sensor2 = (long[]) values2[i]; - sensor2[row2] = i; - long[] sensor3 = (long[]) values3[i]; - sensor3[row3] = i; - } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { - session.insertTablets(tabletMap, true); - - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - } - - if (tablet1.rowSize != 0) { + if (tablet1.getRowSize() != 0) { session.insertTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -641,14 +539,14 @@ private static void insertText() throws IoTDBConnectionException, StatementExecu } // insertTablet example - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s2", TSDataType.TEXT)); Tablet tablet = new Tablet(device, schemaList, 100); for (int i = 0; i < datas.size(); i++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, i); // write data of String type or Binary type - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, datas.get(i)); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, datas.get(i)); } session.insertTablet(tablet); try (SessionDataSet dataSet = session.executeQueryStatement("select s1, s2 from " + device)) { diff --git a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java index 1ec74a34a29ca..6f4d503bce0f2 100644 --- a/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java @@ -24,11 +24,11 @@ import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.subscription.SubscriptionSession; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; import org.apache.iotdb.session.subscription.consumer.AckStrategy; import org.apache.iotdb.session.subscription.consumer.ConsumeResult; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; @@ -122,7 +122,8 @@ private static void dataQuery() throws Exception { /** single pull consumer subscribe topic with path and time range */ private static void dataSubscription1() throws Exception { // Create topics - try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + try (final SubscriptionTreeSession subscriptionSession = + new SubscriptionTreeSession(HOST, PORT)) { subscriptionSession.open(); final Properties config = new Properties(); config.put(TopicConstant.PATH_KEY, "root.db.d1.s1"); @@ -136,7 +137,7 @@ private static void dataSubscription1() throws Exception { final Properties config = new Properties(); config.put(ConsumerConstant.CONSUMER_ID_KEY, "c1"); config.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, "cg1"); - try (SubscriptionPullConsumer consumer1 = new SubscriptionPullConsumer(config)) { + try (final SubscriptionTreePullConsumer consumer1 = new SubscriptionTreePullConsumer(config)) { consumer1.open(); consumer1.subscribe(TOPIC_1); while (true) { @@ -160,7 +161,8 @@ private static void dataSubscription1() throws Exception { } // Show topics and subscriptions - try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + try (final SubscriptionTreeSession subscriptionSession = + new SubscriptionTreeSession(HOST, PORT)) { subscriptionSession.open(); subscriptionSession.getTopics().forEach((System.out::println)); subscriptionSession.getSubscriptions().forEach((System.out::println)); @@ -172,7 +174,8 @@ private static void dataSubscription1() throws Exception { /** multi pull consumer subscribe topic with tsfile format */ private static void dataSubscription2() throws Exception { - try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + try (final SubscriptionTreeSession subscriptionSession = + new SubscriptionTreeSession(HOST, PORT)) { subscriptionSession.open(); final Properties config = new Properties(); config.put(TopicConstant.START_TIME_KEY, CURRENT_TIME + 33); @@ -189,8 +192,8 @@ private static void dataSubscription2() throws Exception { () -> { int retryCount = 0; // Subscription: builder-style ctor - try (final SubscriptionPullConsumer consumer2 = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer2 = + new SubscriptionTreePullConsumer.Builder() .consumerId("c" + idx) .consumerGroupId("cg2") .autoCommit(false) @@ -238,7 +241,8 @@ private static void dataSubscription2() throws Exception { /** multi push consumer subscribe topic with tsfile format and snapshot mode */ private static void dataSubscription3() throws Exception { - try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + try (final SubscriptionTreeSession subscriptionSession = + new SubscriptionTreeSession(HOST, PORT)) { subscriptionSession.open(); final Properties config = new Properties(); config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); @@ -253,8 +257,8 @@ private static void dataSubscription3() throws Exception { new Thread( () -> { // Subscription: builder-style ctor - try (final SubscriptionPushConsumer consumer3 = - new SubscriptionPushConsumer.Builder() + try (final SubscriptionTreePushConsumer consumer3 = + new SubscriptionTreePushConsumer.Builder() .consumerId("c" + idx) .consumerGroupId("cg3") .ackStrategy(AckStrategy.AFTER_CONSUME) @@ -268,7 +272,7 @@ private static void dataSubscription3() throws Exception { .buildPushConsumer()) { consumer3.open(); consumer3.subscribe(TOPIC_3); - while (!consumer3.allSnapshotTopicMessagesHaveBeenConsumed()) { + while (!consumer3.allTopicMessagesHaveBeenConsumed()) { LockSupport.parkNanos(SLEEP_NS); // wait some time } } @@ -284,7 +288,8 @@ private static void dataSubscription3() throws Exception { /** multi pull consumer subscribe topic with tsfile format and snapshot mode */ private static void dataSubscription4() throws Exception { - try (final SubscriptionSession subscriptionSession = new SubscriptionSession(HOST, PORT)) { + try (final SubscriptionTreeSession subscriptionSession = + new SubscriptionTreeSession(HOST, PORT)) { subscriptionSession.open(); final Properties config = new Properties(); config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); @@ -300,8 +305,8 @@ private static void dataSubscription4() throws Exception { new Thread( () -> { // Subscription: builder-style ctor - try (final SubscriptionPullConsumer consumer4 = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer4 = + new SubscriptionTreePullConsumer.Builder() .consumerId("c" + idx) .consumerGroupId("cg4") .autoCommit(true) @@ -309,7 +314,7 @@ private static void dataSubscription4() throws Exception { .buildPullConsumer()) { consumer4.open(); consumer4.subscribe(TOPIC_4); - while (!consumer4.allSnapshotTopicMessagesHaveBeenConsumed()) { + while (!consumer4.allTopicMessagesHaveBeenConsumed()) { for (final SubscriptionMessage message : consumer4.poll(POLL_TIMEOUT_MS)) { final SubscriptionTsFileHandler handler = message.getTsFileHandler(); handler.moveFile( diff --git a/example/session/src/main/java/org/apache/iotdb/TableModelSessionExample.java b/example/session/src/main/java/org/apache/iotdb/TableModelSessionExample.java new file mode 100644 index 0000000000000..f7e9275ce1589 --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/TableModelSessionExample.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class TableModelSessionExample { + + private static final String LOCAL_URL = "127.0.0.1:6667"; + + public static void main(String[] args) { + + // don't specify database in constructor + try (ITableSession session = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(LOCAL_URL)) + .username("root") + .password("root") + .build()) { + + session.executeNonQueryStatement("CREATE DATABASE test1"); + session.executeNonQueryStatement("CREATE DATABASE test2"); + + session.executeNonQueryStatement("use test2"); + + // or use full qualified table name + session.executeNonQueryStatement( + "create table test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + + session.executeNonQueryStatement( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD) with (TTL=6600000)"); + + // show tables from current database + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + // show tables by specifying another database + // using SHOW tables FROM + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES FROM test1")) { + System.out.println(dataSet.getColumnNames()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "model", "temperature", "humidity"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.DOUBLE); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("test1", columnNameList, dataTypeList, columnTypeList, 100); + for (long timestamp = 0; timestamp < 100; timestamp++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("region_id", rowIndex, "1"); + tablet.addValue("plant_id", rowIndex, "5"); + tablet.addValue("device_id", rowIndex, "3"); + tablet.addValue("model", rowIndex, "A"); + tablet.addValue("temperature", rowIndex, 37.6F); + tablet.addValue("humidity", rowIndex, 111.1); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + // query table data + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select * from test1 " + + "where region_id = '1' and plant_id in ('3', '5') and device_id = '3'")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } + + // specify database in constructor + try (ITableSession session = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(LOCAL_URL)) + .username("root") + .password("root") + .database("test1") + .build()) { + + // show tables from current database + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + // change database to test2 + session.executeNonQueryStatement("use test2"); + + // show tables by specifying another database + // using SHOW tables FROM + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } + } +} diff --git a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java new file mode 100644 index 0000000000000..a8d3c8b382b3b --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class TableModelSessionPoolExample { + + private static final String LOCAL_URL = "127.0.0.1:6667"; + + public static void main(String[] args) { + + // don't specify database in constructor + ITableSessionPool tableSessionPool = + new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(LOCAL_URL)) + .user("root") + .password("root") + .maxSize(1) + .build(); + + try (ITableSession session = tableSessionPool.getSession()) { + + session.executeNonQueryStatement("CREATE DATABASE test1"); + session.executeNonQueryStatement("CREATE DATABASE test2"); + + session.executeNonQueryStatement("use test2"); + + // or use full qualified table name + session.executeNonQueryStatement( + "create table test1.table1(" + + "region_id STRING TAG, " + + "plant_id STRING TAG, " + + "device_id STRING TAG, " + + "model STRING ATTRIBUTE, " + + "temperature FLOAT FIELD, " + + "humidity DOUBLE FIELD) with (TTL=3600000)"); + + session.executeNonQueryStatement( + "create table table2(" + + "region_id STRING TAG, " + + "plant_id STRING TAG, " + + "color STRING ATTRIBUTE, " + + "temperature FLOAT FIELD, " + + "speed DOUBLE FIELD) with (TTL=6600000)"); + + // show tables from current database + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + // show tables by specifying another database + // using SHOW tables FROM + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES FROM test1")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "model", "temperature", "humidity"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.DOUBLE); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("test1", columnNameList, dataTypeList, columnTypeList, 100); + for (long timestamp = 0; timestamp < 100; timestamp++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("region_id", rowIndex, "1"); + tablet.addValue("plant_id", rowIndex, "5"); + tablet.addValue("device_id", rowIndex, "3"); + tablet.addValue("model", rowIndex, "A"); + tablet.addValue("temperature", rowIndex, 37.6F); + tablet.addValue("humidity", rowIndex, 111.1); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + // query table data + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select * from test1 " + + "where region_id = '1' and plant_id in ('3', '5') and device_id = '3'")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } finally { + tableSessionPool.close(); + } + + // specify database in constructor + tableSessionPool = + new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(LOCAL_URL)) + .user("root") + .password("root") + .maxSize(1) + .database("test1") + .build(); + + try (ITableSession session = tableSessionPool.getSession()) { + + // show tables from current database + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + // change database to test2 + session.executeNonQueryStatement("use test2"); + + // show tables by specifying another database + // using SHOW tables FROM + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } + + try (ITableSession session = tableSessionPool.getSession()) { + + // show tables from default database test1 + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } finally { + tableSessionPool.close(); + } + } +} diff --git a/example/session/src/main/java/org/apache/iotdb/TableModelSubscriptionSessionExample.java b/example/session/src/main/java/org/apache/iotdb/TableModelSubscriptionSessionExample.java new file mode 100644 index 0000000000000..363ff971662d3 --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/TableModelSubscriptionSessionExample.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.TableSessionBuilder; +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +public class TableModelSubscriptionSessionExample { + + private static final String HOST = "127.0.0.1"; + private static final int PORT = 6667; + + private static final String TOPIC_1 = "topic1"; + private static final String TOPIC_2 = "`'topic2'`"; + private static final String TOPIC_3 = "`\"topic3\"`"; + private static final String TOPIC_4 = "`\"top \\.i.c4\"`"; + + private static final long SLEEP_NS = 1_000_000_000L; + private static final long POLL_TIMEOUT_MS = 10_000L; + private static final int MAX_RETRY_TIMES = 3; + private static final int PARALLELISM = 8; + private static final long CURRENT_TIME = System.currentTimeMillis(); + + private static void createDataBaseAndTable( + final ITableSession session, final String database, final String table) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement("create database if not exists " + database); + session.executeNonQueryStatement("use " + database); + session.executeNonQueryStatement( + String.format( + "CREATE TABLE %s (s0 string tag, s1 string tag, s2 string tag, s3 string tag, s4 int64 field, s5 float field, s6 string field, s7 timestamp field, s8 int32 field, s9 double field, s10 date field, s11 text field)", + table)); + } + + private static void insertData( + final ITableSession session, + final String dataBaseName, + final String tableName, + final int start, + final int end) + throws IoTDBConnectionException, StatementExecutionException { + final List list = new ArrayList<>(end - start + 1); + for (int i = start; i < end; ++i) { + list.add( + String.format( + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); + } + list.add("flush"); + session.executeNonQueryStatement("use " + dataBaseName); + for (final String s : list) { + session.executeNonQueryStatement(s); + } + } + + private static String getDateStr(final int value) { + final Date date = new Date(value); + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try { + return dateFormat.format(date); + } catch (final Exception e) { + return "1970-01-01"; + } + } + + private static void dataSubscription() throws Exception { + // Create topics + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host("127.0.0.1") + .port(6667) + .username("root") + .password("root") + .build()) { + final Properties config = new Properties(); + config.put(TopicConstant.DATABASE_KEY, "db.*"); + config.put(TopicConstant.TABLE_KEY, "test.*"); + config.put(TopicConstant.START_TIME_KEY, 25); + config.put(TopicConstant.END_TIME_KEY, 75); + config.put(TopicConstant.STRICT_KEY, "true"); + session.createTopic(TOPIC_1, config); + } + + int retryCount = 0; + try (final ISubscriptionTablePullConsumer consumer1 = + new SubscriptionTablePullConsumerBuilder() + .consumerId("c1") + .consumerGroupId("cg1") + .build()) { + consumer1.open(); + consumer1.subscribe(TOPIC_1); + while (true) { + final List messages = consumer1.poll(POLL_TIMEOUT_MS); + if (messages.isEmpty()) { + retryCount++; + if (retryCount >= MAX_RETRY_TIMES) { + break; + } + } + for (final SubscriptionMessage message : messages) { + for (final SubscriptionSessionDataSet dataSet : message.getSessionDataSetsHandler()) { + System.out.println(dataSet.getDatabaseName()); + System.out.println(dataSet.getTableName()); + System.out.println(dataSet.getColumnNames()); + System.out.println(dataSet.getColumnTypes()); + System.out.println(dataSet.getColumnCategories()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + // Auto commit + } + + // Show topics and subscriptions + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host("127.0.0.1") + .port(6667) + .username("root") + .password("root") + .build()) { + session.getTopics().forEach((System.out::println)); + session.getSubscriptions().forEach((System.out::println)); + } + + consumer1.unsubscribe(TOPIC_1); + } + } + + public static void main(final String[] args) throws Exception { + try (final ITableSession session = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(HOST + ":" + PORT)) + .username("root") + .password("root") + .build()) { + createDataBaseAndTable(session, "db1", "test1"); + createDataBaseAndTable(session, "db1", "test2"); + createDataBaseAndTable(session, "db2", "test1"); + createDataBaseAndTable(session, "db2", "test2"); + insertData(session, "db1", "test1", 0, 100); + insertData(session, "db1", "test2", 0, 100); + insertData(session, "db2", "test1", 0, 100); + insertData(session, "db2", "test2", 0, 100); + dataSubscription(); + } + } +} diff --git a/example/session/src/main/java/org/apache/iotdb/TabletExample.java b/example/session/src/main/java/org/apache/iotdb/TabletExample.java index 2b2d595e2e844..f85dba2215b77 100644 --- a/example/session/src/main/java/org/apache/iotdb/TabletExample.java +++ b/example/session/src/main/java/org/apache/iotdb/TabletExample.java @@ -24,6 +24,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.BufferedReader; @@ -89,9 +90,11 @@ private static Map> loadCSVData( case BOOLEAN: ret.get(measurement).add(Boolean.parseBoolean(items[idx])); break; + case DATE: case INT32: ret.get(measurement).add(Integer.parseInt(items[idx])); break; + case TIMESTAMP: case INT64: ret.get(measurement).add(Long.parseLong(items[idx])); break; @@ -101,6 +104,8 @@ private static Map> loadCSVData( case DOUBLE: ret.get(measurement).add(Double.parseDouble(items[idx])); break; + case STRING: + case BLOB: case TEXT: ret.get(measurement).add(BytesUtils.valueOf(items[idx])); break; @@ -148,7 +153,7 @@ public static void main(String[] args) throws Exception { measureTSTypeInfos.put("s3", TSDataType.DOUBLE); measureTSTypeInfos.put("s4", TSDataType.INT64); measureTSTypeInfos.put("s5", TSDataType.TEXT); - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); measureTSTypeInfos.forEach((mea, type) -> schemas.add(new MeasurementSchema(mea, type))); System.out.println( @@ -167,8 +172,7 @@ public static void main(String[] args) throws Exception { String deviceId = "root.sg" + i % 8 + "." + i; Tablet ta = new Tablet(deviceId, schemas, rowSize); - ta.rowSize = rowSize; - for (int t = 0; t < ta.rowSize; t++) { + for (int t = 0; t < rowSize; t++) { ta.addTimestamp(t, (Long) data.get(TIME_STR).get(t)); for (Entry entry : measureTSTypeInfos.entrySet()) { String mea = entry.getKey(); diff --git a/example/trigger/pom.xml b/example/trigger/pom.xml index 1970972306f4f..5951e0e011225 100644 --- a/example/trigger/pom.xml +++ b/example/trigger/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT trigger-example IoTDB: Example: Trigger diff --git a/example/trigger/src/main/java/org/apache/iotdb/trigger/LoggerTrigger.java b/example/trigger/src/main/java/org/apache/iotdb/trigger/LoggerTrigger.java index f1b5730e3e6b8..bb95f62de083f 100644 --- a/example/trigger/src/main/java/org/apache/iotdb/trigger/LoggerTrigger.java +++ b/example/trigger/src/main/java/org/apache/iotdb/trigger/LoggerTrigger.java @@ -24,7 +24,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,20 +36,20 @@ public class LoggerTrigger implements Trigger { @Override public boolean fire(Tablet tablet) throws Exception { - List measurementSchemaList = tablet.getSchemas(); + List measurementSchemaList = tablet.getSchemas(); for (int i = 0, n = measurementSchemaList.size(); i < n; i++) { if (measurementSchemaList.get(i).getType().equals(TSDataType.DOUBLE)) { - logDouble((double[]) tablet.values[i]); + logDouble((double[]) tablet.getValues()[i]); } else if (measurementSchemaList.get(i).getType().equals(TSDataType.FLOAT)) { - logFloat((float[]) tablet.values[i]); + logFloat((float[]) tablet.getValues()[i]); } else if (measurementSchemaList.get(i).getType().equals(TSDataType.INT64)) { - logLong((long[]) tablet.values[i]); + logLong((long[]) tablet.getValues()[i]); } else if (measurementSchemaList.get(i).getType().equals(TSDataType.INT32)) { - logInt((int[]) tablet.values[i]); + logInt((int[]) tablet.getValues()[i]); } else if (measurementSchemaList.get(i).getType().equals(TSDataType.TEXT)) { - logText((Binary[]) tablet.values[i]); + logText((Binary[]) tablet.getValues()[i]); } else if (measurementSchemaList.get(i).getType().equals(TSDataType.BOOLEAN)) { - logBoolean((boolean[]) tablet.values[i]); + logBoolean((boolean[]) tablet.getValues()[i]); } } return true; diff --git a/example/trigger/src/main/java/org/apache/iotdb/trigger/StatisticsUpdaterTrigger.java b/example/trigger/src/main/java/org/apache/iotdb/trigger/StatisticsUpdaterTrigger.java index 6d9f260175a42..4f1217b6f66db 100644 --- a/example/trigger/src/main/java/org/apache/iotdb/trigger/StatisticsUpdaterTrigger.java +++ b/example/trigger/src/main/java/org/apache/iotdb/trigger/StatisticsUpdaterTrigger.java @@ -81,16 +81,16 @@ public void onCreate(TriggerAttributes attributes) throws Exception { @Override public boolean fire(Tablet tablet) throws Exception { ensureSession(); - if (tablet.bitMaps == null) { - cnt.addAndGet((long) tablet.rowSize * tablet.getSchemas().size()); + if (tablet.getBitMaps() == null) { + cnt.addAndGet((long) tablet.getRowSize() * tablet.getSchemas().size()); return true; } for (int column = 0; column < tablet.getSchemas().size(); column++) { - BitMap bitMap = tablet.bitMaps[column]; + BitMap bitMap = tablet.getBitMaps()[column]; if (bitMap == null) { - cnt.addAndGet(tablet.rowSize); + cnt.addAndGet(tablet.getRowSize()); } else { - for (int row = 0; row < tablet.rowSize; row++) { + for (int row = 0; row < tablet.getRowSize(); row++) { if (!bitMap.isMarked(row)) { cnt.incrementAndGet(); } diff --git a/example/udf/pom.xml b/example/udf/pom.xml index db0cd00884594..48b7f1a02f1b9 100644 --- a/example/udf/pom.xml +++ b/example/udf/pom.xml @@ -24,14 +24,18 @@ org.apache.iotdb iotdb-examples - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT udf-example IoTDB: Example: UDF + + org.slf4j + slf4j-api + org.apache.iotdb - iotdb-server + udf-api ${project.version} provided diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java new file mode 100644 index 0000000000000..4837e7e97f020 --- /dev/null +++ b/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf; + +import org.apache.iotdb.udf.api.State; +import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.relational.AggregateFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; +import org.apache.iotdb.udf.api.utils.ResultValue; + +import java.nio.ByteBuffer; + +/** + * This is an internal example of the AggregateFunction implementation. + * + *

CREATE DATABASE test; + * + *

USE test; + * + *

CREATE TABLE t1(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD); + * + *

INSERT INTO t1(time, device_id, s1, s2) VALUES (1, 'd1', 'a', 1), (2, 'd1', null, 2), (3, + * 'd2', 'c', null); + * + *

CREATE FUNCTION my_count AS 'org.apache.iotdb.udf.AggregateFunctionExample'; + * + *

SHOW FUNCTIONS; + * + *

SELECT device_id, my_count(s1) as s1_count, my_count(s2) as s2_count FROM t1 group by + * device_id; + * + *

SELECT my_count(s1) as s1_count, my_count(s2) as s2_count FROM t1; + */ +public class AggregateFunctionExample implements AggregateFunction { + + static class CountState implements State { + + long count; + + @Override + public void reset() { + count = 0; + } + + @Override + public byte[] serialize() { + ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES); + buffer.putLong(count); + return buffer.array(); + } + + @Override + public void deserialize(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + count = buffer.getLong(); + } + } + + @Override + public AggregateFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() != 1) { + throw new UDFArgumentNotValidException("Only one parameter is required."); + } + return new AggregateFunctionAnalysis.Builder() + .outputDataType(Type.INT64) + .removable(true) + .build(); + } + + @Override + public State createState() { + return new CountState(); + } + + @Override + public void addInput(State state, Record input) { + CountState countState = (CountState) state; + if (!input.isNull(0)) { + countState.count++; + } + } + + @Override + public void combineState(State state, State rhs) { + CountState countState = (CountState) state; + CountState rhsCountState = (CountState) rhs; + countState.count += rhsCountState.count; + } + + @Override + public void outputFinal(State state, ResultValue resultValue) { + CountState countState = (CountState) state; + resultValue.setLong(countState.count); + } + + @Override + public void remove(State state, Record input) { + CountState countState = (CountState) state; + if (!input.isNull(0)) { + countState.count--; + } + } +} diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java new file mode 100644 index 0000000000000..ff48d0b57a9a2 --- /dev/null +++ b/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf; + +import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; + +/** + * This is an internal example of the ScalarFunction implementation. + * + *

CREATE DATABASE test; + * + *

USE test; + * + *

CREATE TABLE t1(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD); + * + *

INSERT INTO t1(time, device_id, s1, s2) VALUES (1, 'd1', 'a', 1), (2, 'd1', null, 2), (3, + * 'd1', 'c', null); + * + *

CREATE FUNCTION contain_null AS 'org.apache.iotdb.udf.ScalarFunctionExample'; + * + *

SHOW FUNCTIONS; + * + *

SELECT time, device_id, s1, s2, contain_null(s1, s2) as contain_null, contain_null(s1) as + * s1_isnull, contain_null(s2) as s2_isnull FROM t1; + */ +public class ScalarFunctionExample implements ScalarFunction { + + @Override + public ScalarFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() < 1) { + throw new UDFArgumentNotValidException("At least one parameter is required."); + } + return new ScalarFunctionAnalysis.Builder().outputDataType(Type.BOOLEAN).build(); + } + + @Override + public Object evaluate(Record input) throws UDFException { + for (int i = 0; i < input.size(); i++) { + if (input.isNull(i)) { + return true; + } + } + return false; + } +} diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java index b2b9bec5f1bab..540f6bcef5cce 100644 --- a/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java +++ b/example/udf/src/main/java/org/apache/iotdb/udf/UDAFExample.java @@ -110,6 +110,10 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { return; case TEXT: case BOOLEAN: + case TIMESTAMP: + case STRING: + case BLOB: + case DATE: default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in aggregation AVG : %s", dataType)); diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/table/ExcludeColumnExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/table/ExcludeColumnExample.java new file mode 100644 index 0000000000000..b1cdf810e414b --- /dev/null +++ b/example/udf/src/main/java/org/apache/iotdb/udf/table/ExcludeColumnExample.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.table; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.EmptyTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * This is an internal example of the TableFunction implementation. This function is declared as row + * semantic without pass through columns. + * + *

CREATE DATABASE test; + * + *

USE test; + * + *

CREATE TABLE t1(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD); + * + *

INSERT INTO t1(time, device_id, s1, s2) VALUES (1, 'd1', 'a', 1), (2, 'd1', null, 2), (3, + * 'd1', 'c', null); + * + *

CREATE FUNCTION exclude_column AS 'org.apache.iotdb.udf.table.ExcludeColumnExample'; + * + *

SHOW FUNCTIONS; + * + *

SELECT * FROM exclude_column(t1, 's2'); + */ +public class ExcludeColumnExample implements TableFunction { + private final String TBL_PARAM = "DATA"; + private final String COL_PARAM = "EXCLUDE"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder().name(TBL_PARAM).rowSemantics().build(), + ScalarParameterSpecification.builder().name(COL_PARAM).type(Type.STRING).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + TableArgument tableArgument = (TableArgument) arguments.get(TBL_PARAM); + String excludeColumn = (String) ((ScalarArgument) arguments.get(COL_PARAM)).getValue(); + List requiredColumns = new ArrayList<>(); + DescribedSchema.Builder schemaBuilder = DescribedSchema.builder(); + for (int i = 0; i < tableArgument.getFieldNames().size(); i++) { + Optional fieldName = tableArgument.getFieldNames().get(i); + if (!fieldName.isPresent() || !fieldName.get().equalsIgnoreCase(excludeColumn)) { + requiredColumns.add(i); + schemaBuilder.addField(fieldName, tableArgument.getFieldTypes().get(i)); + } + } + return TableFunctionAnalysis.builder() + .properColumnSchema(schemaBuilder.build()) + .requiredColumns(TBL_PARAM, requiredColumns) + .handle(new EmptyTableFunctionHandle()) + .build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new EmptyTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return (input, properColumnBuilders, passThroughIndexBuilder) -> { + for (int i = 0; i < input.size(); i++) { + if (input.isNull(i)) { + properColumnBuilders.get(i).appendNull(); + } else { + properColumnBuilders.get(i).writeObject(input.getObject(i)); + } + } + }; + } + }; + } +} diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/table/RepeatExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/table/RepeatExample.java new file mode 100644 index 0000000000000..7360900ba80b7 --- /dev/null +++ b/example/udf/src/main/java/org/apache/iotdb/udf/table/RepeatExample.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.table; + +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * This is an internal example of the TableFunction implementation. This function is declared as set + * semantic with pass through columns. + * + *

CREATE DATABASE test; + * + *

USE test; + * + *

CREATE TABLE t1(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD); + * + *

INSERT INTO t1(time, device_id, s1, s2) VALUES (1, 'd1', 'a', 1), (2, 'd1', null, 2), (3, + * 'd1', 'c', null); + * + *

CREATE FUNCTION repeat AS 'org.apache.iotdb.udf.table.RepeatExample'; + * + *

SHOW FUNCTIONS; + * + *

SELECT * FROM repeat(t1, 2); + */ +public class RepeatExample implements TableFunction { + private final String TBL_PARAM = "DATA"; + private final String N_PARAM = "N"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder().name(TBL_PARAM).passThroughColumns().build(), + ScalarParameterSpecification.builder().name(N_PARAM).type(Type.INT32).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + + ScalarArgument count = (ScalarArgument) arguments.get("N"); + if (count == null) { + throw new UDFArgumentNotValidException("count argument for function repeat() is missing"); + } else if ((int) count.getValue() <= 0) { + throw new UDFArgumentNotValidException( + "count argument for function repeat() must be positive"); + } + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder().addProperty(N_PARAM, count.getValue()).build(); + return TableFunctionAnalysis.builder() + .properColumnSchema(DescribedSchema.builder().addField("repeat_index", Type.INT32).build()) + .requiredColumns( + TBL_PARAM, + Collections.singletonList(0)) // per spec, function must require at least one column + .handle(handle) + .build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return new TableFunctionDataProcessor() { + private final int n = + (int) ((MapTableFunctionHandle) tableFunctionHandle).getProperty(N_PARAM); + private long recordIndex = 0; + + @Override + public void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder) { + properColumnBuilders.get(0).writeInt(0); + passThroughIndexBuilder.writeLong(recordIndex++); + } + + @Override + public void finish( + List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { + for (int i = 1; i < n; i++) { + for (int j = 0; j < recordIndex; j++) { + properColumnBuilders.get(0).writeInt(i); + passThroughIndexBuilder.writeLong(j); + } + } + } + }; + } + }; + } +} diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/table/SplitExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/table/SplitExample.java new file mode 100644 index 0000000000000..03c46ad4a0863 --- /dev/null +++ b/example/udf/src/main/java/org/apache/iotdb/udf/table/SplitExample.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.table; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionLeafProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.utils.Binary; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * This is an internal example of the TableFunction implementation. This function is leaf table + * function without table parameter. You can use this function to split a string into multiple rows. + * + *

CREATE DATABASE test; + * + *

USE test; + * + *

CREATE FUNCTION split AS 'org.apache.iotdb.udf.table.SplitExample'; + * + *

SHOW FUNCTIONS; + * + *

SELECT * FROM TABLE(split('1,2,3,4,5')); + * + *

SELECT * FROM split('1、2、3、4、5', '、'); + */ +public class SplitExample implements TableFunction { + private final String INPUT_PARAMETER_NAME = "INPUT"; + private final String SPLIT_PARAMETER_NAME = "SPLIT"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + ScalarParameterSpecification.builder().name(INPUT_PARAMETER_NAME).type(Type.STRING).build(), + ScalarParameterSpecification.builder() + .name(SPLIT_PARAMETER_NAME) + .type(Type.STRING) + .defaultValue(",") + .build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + DescribedSchema schema = DescribedSchema.builder().addField("output", Type.STRING).build(); + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder() + .addProperty( + INPUT_PARAMETER_NAME, + ((ScalarArgument) arguments.get(INPUT_PARAMETER_NAME)).getValue()) + .addProperty( + SPLIT_PARAMETER_NAME, + ((ScalarArgument) arguments.get(SPLIT_PARAMETER_NAME)).getValue()) + .build(); + return TableFunctionAnalysis.builder().properColumnSchema(schema).handle(handle).build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionLeafProcessor getSplitProcessor() { + return new SplitProcessor( + (String) + ((MapTableFunctionHandle) tableFunctionHandle).getProperty(INPUT_PARAMETER_NAME), + (String) + ((MapTableFunctionHandle) tableFunctionHandle).getProperty(SPLIT_PARAMETER_NAME)); + } + }; + } + + private static class SplitProcessor implements TableFunctionLeafProcessor { + private final String input; + private final String split; + private boolean finish = false; + + SplitProcessor(String input, String split) { + this.input = input; + this.split = split; + } + + @Override + public void process(List columnBuilders) { + for (String s : input.split(split)) { + columnBuilders.get(0).writeBinary(new Binary(s, Charset.defaultCharset())); + } + finish = true; + } + + @Override + public boolean isFinish() { + return finish; + } + } +} diff --git a/integration-test/README.md b/integration-test/README.md index 2514cce9030d9..16b2becf1751b 100644 --- a/integration-test/README.md +++ b/integration-test/README.md @@ -23,10 +23,12 @@ Integration tests for IoTDB are in this module. -Now integration testing supports two kinds of architecture. +Now integration testing supports four kinds of architecture. -- `Simple`: A cluster with 1 config node and 1 data node. -- `Cluster1`: A cluster with 1 config node and 3 data nodes. +- `Simple`: A cluster with 1 config node and 1 data node running tree model ITs. +- `Cluster1`: A cluster with 1 config node and 3 data nodes running tree model ITs. +- `TABLE_SIMPLE`: A cluster with 1 config node and 1 data node running table model ITs. +- `TABLE_CLUSTER1`: A cluster with 1 config node and 3 data nodes running table model ITs. ## Integration Testing with Simple Consensus Mode @@ -51,7 +53,7 @@ After doing this, you can run any one just by clicking the test case and pressin ## Integration Testing with Cluster Mode -You can run the integration test in a 'real' cluster mode. At present, we have implemented a pseudo cluster with 1 config nodes and 3 data nodes. +You can run the integration test in a 'real' cluster mode for tree model ITs. At present, we have implemented a pseudo cluster with 1 config nodes and 3 data nodes. (As the test cases and the test environment are decoupled, we can easily implement other pseudo cluster or even a docker-based cluster later.) The maven command is: @@ -59,6 +61,20 @@ The maven command is: mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT -P with-integration-tests ``` +You can run the integration test in `Simple` mode for table model ITs. +The maven command is: +``` +mvn clean verify -DskipUTs -pl integration-test -am -PTableSimpleIT -P with-integration-tests +``` + +You can run the integration test in 'real' cluster mode for table model ITs. At present, we have implemented a pseudo cluster with 1 config nodes and 3 data nodes. +(As the test cases and the test environment are decoupled, we can easily implement other pseudo cluster or even a docker-based cluster later.) +The maven command is: +``` +mvn clean verify -DskipUTs -pl integration-test -am -PTableClusterIT -P with-integration-tests +``` + + If you want to run IT in `Cluster1` mode in the IDE like IntelliJ, you need to achieve the effect as the `ClusterIT` profile in maven explicitly. Follow Steps 1-4 to achieve it. diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 896a9bfa94085..f8a1bd76a3323 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT integration-test IoTDB: Integration-Test @@ -37,6 +37,7 @@ 200m true true + 0 Simple @@ -67,52 +68,66 @@ 3 Ratis 3 + Ratis + 1 + 3 + IoTV2 + 2 + Ratis + 3 + Ratis + 1 + 3 + IoTV2 + 2 + Ratis + 3 org.apache.iotdb iotdb-server - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-session - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-jdbc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb trigger-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb isession - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb service-rpc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb node-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.tsfile @@ -122,7 +137,7 @@ org.apache.iotdb udf-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb @@ -132,7 +147,7 @@ org.apache.iotdb iotdb-consensus - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.slf4j @@ -161,17 +176,17 @@ org.apache.iotdb iotdb-confignode - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-cli - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT commons-codec @@ -201,7 +216,7 @@ org.apache.iotdb iotdb-server - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT test-jar test @@ -210,6 +225,10 @@ jcip-annotations test + + org.fusesource.mqtt-client + mqtt-client + @@ -286,6 +305,7 @@ ${integrationTest.testEnv} ${integrationTest.randomSelectWriteNode} ${integrationTest.readAndVerifyWithMultiNode} + ${integrationTest.dataRegionPerDataNode} ${lightWeightStandaloneMode.configNodeNumber} ${lightWeightStandaloneMode.dataNodeNumber} ${lightWeightStandaloneMode.configNodeConsensus} @@ -314,6 +334,20 @@ ${strongConsistencyClusterMode.dataRegionConsensus} ${strongConsistencyClusterMode.schemaRegionReplicaNumber} ${strongConsistencyClusterMode.dataRegionReplicaNumber} + ${pipeConsensusBatchMode.configNodeNumber} + ${pipeConsensusBatchMode.dataNodeNumber} + ${pipeConsensusBatchMode.configNodeConsensus} + ${pipeConsensusBatchMode.schemaRegionConsensus} + ${pipeConsensusBatchMode.dataRegionConsensus} + ${pipeConsensusBatchMode.schemaRegionReplicaNumber} + ${pipeConsensusBatchMode.dataRegionReplicaNumber} + ${pipeConsensusStreamMode.configNodeNumber} + ${pipeConsensusStreamMode.dataNodeNumber} + ${pipeConsensusStreamMode.configNodeConsensus} + ${pipeConsensusStreamMode.schemaRegionConsensus} + ${pipeConsensusStreamMode.dataRegionConsensus} + ${pipeConsensusStreamMode.schemaRegionReplicaNumber} + ${pipeConsensusStreamMode.dataRegionReplicaNumber} target/failsafe-reports/failsafe-summary-IT.xml @@ -388,7 +422,7 @@ true - + org.apache.iotdb.itbase.category.ManualIT org.apache.iotdb.itbase.category.LocalStandaloneIT false false @@ -403,7 +437,7 @@ false - + org.apache.iotdb.itbase.category.ManualIT org.apache.iotdb.itbase.category.RemoteIT Remote @@ -414,8 +448,8 @@ false - - org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2Subscription,org.apache.iotdb.itbase.category.MultiClusterIT3 + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableRegressionConsumer,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableRegressionMisc,org.apache.iotdb.itbase.category.MultiClusterIT3,org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic,org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced false true true @@ -428,7 +462,7 @@ false - + org.apache.iotdb.itbase.category.ManualIT org.apache.iotdb.itbase.category.MultiClusterIT1 false true @@ -437,13 +471,13 @@ - MultiClusterIT2AutoCreateSchema + MultiClusterIT2DualTreeAutoBasic false - - org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic false true true @@ -451,13 +485,13 @@ - MultiClusterIT2ManualCreateSchema + MultiClusterIT2DualTreeAutoEnhanced false - - org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced false true true @@ -465,13 +499,125 @@ - MultiClusterIT2Subscription + MultiClusterIT2DualTableManualBasic false - - org.apache.iotdb.itbase.category.MultiClusterIT2Subscription + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic + false + true + true + MultiCluster + + + + MultiClusterIT2DualTableManualEnhanced + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced + false + true + true + MultiCluster + + + + MultiClusterIT2DualTreeManual + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionTreeArchVerification + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeArchVerification + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionTreeRegressionConsumer + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionTreeRegressionMisc + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionTableArchVerification + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableArchVerification + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionTableRegressionConsumer + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableRegressionConsumer + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionTableRegressionMisc + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableRegressionMisc false true true @@ -484,7 +630,7 @@ false - + org.apache.iotdb.itbase.category.ManualIT org.apache.iotdb.itbase.category.MultiClusterIT3 false true @@ -498,7 +644,7 @@ false - + org.apache.iotdb.itbase.category.ManualIT org.apache.iotdb.itbase.category.ClusterIT false true @@ -506,6 +652,20 @@ Cluster1 + + AIClusterIT + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.AIClusterIT + false + false + false + AI + + DailyIT @@ -520,5 +680,47 @@ Cluster1 + + TableSimpleIT + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.TableLocalStandaloneIT + false + false + false + TABLE_SIMPLE + + + + TableClusterIT + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.TableClusterIT + false + true + true + TABLE_CLUSTER1 + + + + ManualIT + + false + + + + org.apache.iotdb.itbase.category.ManualIT + false + true + true + Simple + + diff --git a/integration-test/src/assembly/mpp-test.xml b/integration-test/src/assembly/mpp-test.xml index 3dc443c8d03f3..71f184549b2bc 100644 --- a/integration-test/src/assembly/mpp-test.xml +++ b/integration-test/src/assembly/mpp-test.xml @@ -43,36 +43,33 @@ ${project.basedir}/../iotdb-core/metrics/interface/src/main/assembly/resources/conf - sbin - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/sbin + conf + ${project.basedir}/../iotdb-core/ainode/resources/conf + + + conf + ${project.basedir}/../scripts/conf 0755 sbin - ${project.basedir}/../iotdb-core/confignode/src/assembly/resources/sbin + ${project.basedir}/../scripts/sbin 0755 tools - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/tools + ${project.basedir}/../scripts/tools 0755 - sbin - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/sbin + venv + ${project.basedir}/../iotdb-core/ainode/venv 0755 - tools - ${project.basedir}/../iotdb-client/cli/src/assembly/resources/tools + lib + ${project.basedir}/../iotdb-core/ainode/dist/ 0755 - - - ${project.basedir}/../iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh - conf/datanode-env.sh - 0755 - - diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java index a17905e4b1518..a8aab5d0a452c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFAvg.java @@ -101,7 +101,11 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { addDoubleInput(avgState, columns, bitMap); return; case TEXT: + case STRING: + case BLOB: case BOOLEAN: + case TIMESTAMP: + case DATE: default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in aggregation AVG : %s", dataType)); diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java index 087dc655d8444..a829c9f2c477b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/UDAFSum.java @@ -101,7 +101,11 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { addDoubleInput(sumState, columns, bitMap); return; case TEXT: + case STRING: + case BLOB: case BOOLEAN: + case TIMESTAMP: + case DATE: default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in aggregation AVG : %s", dataType)); diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java new file mode 100644 index 0000000000000..2f2527f817546 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; + +import java.util.HashSet; +import java.util.Set; + +/** Calculate the sum of all arguments. Only support inputs of INT32,INT64,DOUBLE,FLOAT type. */ +public class AllSum implements ScalarFunction { + + private Type outputDataType; + + @Override + public ScalarFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() < 1) { + throw new UDFArgumentNotValidException("At least one parameter is required."); + } + for (int i = 0; i < arguments.getArgumentsSize(); i++) { + if (arguments.getDataType(i) != Type.INT32 + && arguments.getDataType(i) != Type.INT64 + && arguments.getDataType(i) != Type.FLOAT + && arguments.getDataType(i) != Type.DOUBLE) { + throw new UDFArgumentNotValidException( + "Only support inputs of INT32,INT64,DOUBLE,FLOAT type."); + } + } + return new ScalarFunctionAnalysis.Builder() + .outputDataType(inferOutputDataType(arguments)) + .build(); + } + + @Override + public void beforeStart(FunctionArguments arguments) throws UDFException { + this.outputDataType = inferOutputDataType(arguments); + } + + private Type inferOutputDataType(FunctionArguments arguments) { + Set inputTypeSet = new HashSet<>(); + for (int i = 0; i < arguments.getArgumentsSize(); i++) { + inputTypeSet.add(arguments.getDataType(i)); + } + if (inputTypeSet.contains(Type.DOUBLE)) { + return Type.DOUBLE; + } else if (inputTypeSet.contains(Type.FLOAT)) { + return Type.FLOAT; + } else if (inputTypeSet.contains(Type.INT64)) { + return Type.INT64; + } else { + return Type.INT32; + } + } + + @Override + public Object evaluate(Record input) { + double res = 0; + for (int i = 0; i < input.size(); i++) { + if (!input.isNull(i)) { + switch (input.getDataType(i)) { + case INT32: + res += input.getInt(i); + break; + case INT64: + res += input.getLong(i); + break; + case FLOAT: + res += input.getFloat(i); + break; + case DOUBLE: + res += input.getDouble(i); + break; + } + } + } + switch (outputDataType) { + case INT32: + return (int) res; + case INT64: + return (long) res; + case FLOAT: + return (float) res; + case DOUBLE: + return res; + default: + throw new RuntimeException("Unexpected output type."); + } + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java new file mode 100644 index 0000000000000..656a79a9c27c9 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; + +public class ContainNull implements ScalarFunction { + + @Override + public ScalarFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() < 1) { + throw new UDFArgumentNotValidException("At least one parameter is required."); + } + return new ScalarFunctionAnalysis.Builder().outputDataType(Type.BOOLEAN).build(); + } + + @Override + public Object evaluate(Record input) { + for (int i = 0; i < input.size(); i++) { + if (input.isNull(i)) { + return true; + } + } + return false; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java new file mode 100644 index 0000000000000..e6fb975830b0a --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; + +import java.time.LocalDate; + +public class DatePlus implements ScalarFunction { + + @Override + public ScalarFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() != 2) { + throw new UDFArgumentNotValidException("Only two parameter is required."); + } + if (arguments.getDataType(0) != Type.DATE) { + throw new UDFArgumentNotValidException("The first parameter should be DATE type."); + } + if (arguments.getDataType(1) != Type.INT32 && arguments.getDataType(1) != Type.INT64) { + throw new UDFArgumentNotValidException("The second parameter should be INT type."); + } + return new ScalarFunctionAnalysis.Builder().outputDataType(Type.DATE).build(); + } + + @Override + public Object evaluate(Record input) { + LocalDate date = input.getLocalDate(0); + int days = input.getInt(1); + return date.plusDays(days); + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java new file mode 100644 index 0000000000000..c2aa402d8283d --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.State; +import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.AggregateFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; +import org.apache.iotdb.udf.api.utils.ResultValue; + +import java.nio.ByteBuffer; + +public class FirstTwoSum implements AggregateFunction { + + static class FirstTwoSumState implements State { + long firstTime = Long.MAX_VALUE; + long secondTime = Long.MAX_VALUE; + double firstValue; + double secondValue; + + @Override + public void reset() { + firstTime = Long.MAX_VALUE; + secondTime = Long.MAX_VALUE; + firstValue = 0; + secondValue = 0; + } + + @Override + public byte[] serialize() { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2 + Double.BYTES * 2); + buffer.putLong(firstTime); + buffer.putLong(secondTime); + buffer.putDouble(firstValue); + buffer.putDouble(secondValue); + return buffer.array(); + } + + @Override + public void deserialize(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + firstTime = buffer.getLong(); + secondTime = buffer.getLong(); + firstValue = buffer.getDouble(); + secondValue = buffer.getDouble(); + } + } + + @Override + public AggregateFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() != 3) { + throw new UDFArgumentNotValidException("FirstTwoSum should accept three column as input"); + } + for (int i = 0; i < 2; i++) { + if (arguments.getDataType(i) != Type.INT32 + && arguments.getDataType(i) != Type.INT64 + && arguments.getDataType(i) != Type.FLOAT + && arguments.getDataType(i) != Type.DOUBLE) { + throw new UDFArgumentNotValidException( + "FirstTwoSum should accept INT32, INT64, FLOAT, DOUBLE as the first two inputs"); + } + } + if (arguments.getDataType(2) != Type.TIMESTAMP) { + throw new UDFArgumentNotValidException( + "FirstTwoSum should accept TIMESTAMP as the third input"); + } + return new AggregateFunctionAnalysis.Builder().outputDataType(Type.DOUBLE).build(); + } + + @Override + public State createState() { + return new FirstTwoSumState(); + } + + @Override + public void addInput(State state, Record input) { + FirstTwoSumState firstTwoSumState = (FirstTwoSumState) state; + long time = input.getLong(2); + if (!input.isNull(0) && time < firstTwoSumState.firstTime) { + firstTwoSumState.firstTime = time; + switch (input.getDataType(0)) { + case INT32: + firstTwoSumState.firstValue = input.getInt(0); + break; + case INT64: + firstTwoSumState.firstValue = input.getLong(0); + break; + case FLOAT: + firstTwoSumState.firstValue = input.getFloat(0); + break; + case DOUBLE: + firstTwoSumState.firstValue = input.getDouble(0); + break; + default: + throw new UDFException( + "FirstTwoSum should accept INT32, INT64, FLOAT, DOUBLE as the first two inputs"); + } + } + if (!input.isNull(1) && time < firstTwoSumState.secondTime) { + firstTwoSumState.secondTime = time; + switch (input.getDataType(1)) { + case INT32: + firstTwoSumState.secondValue = input.getInt(1); + break; + case INT64: + firstTwoSumState.secondValue = input.getLong(1); + break; + case FLOAT: + firstTwoSumState.secondValue = input.getFloat(1); + break; + case DOUBLE: + firstTwoSumState.secondValue = input.getDouble(1); + break; + default: + throw new UDFException( + "FirstTwoSum should accept INT32, INT64, FLOAT, DOUBLE as the first two inputs"); + } + } + } + + @Override + public void combineState(State state, State rhs) { + FirstTwoSumState firstTwoSumState = (FirstTwoSumState) state; + FirstTwoSumState rhsState = (FirstTwoSumState) rhs; + if (rhsState.firstTime < firstTwoSumState.firstTime) { + firstTwoSumState.firstTime = rhsState.firstTime; + firstTwoSumState.firstValue = rhsState.firstValue; + } + if (rhsState.secondTime < firstTwoSumState.secondTime) { + firstTwoSumState.secondTime = rhsState.secondTime; + firstTwoSumState.secondValue = rhsState.secondValue; + } + } + + @Override + public void outputFinal(State state, ResultValue resultValue) { + FirstTwoSumState firstTwoSumState = (FirstTwoSumState) state; + if (firstTwoSumState.firstTime == Long.MAX_VALUE + && firstTwoSumState.secondTime == Long.MAX_VALUE) { + resultValue.setNull(); + } else { + resultValue.setDouble(firstTwoSumState.firstValue + firstTwoSumState.secondValue); + } + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java new file mode 100644 index 0000000000000..94c45254920f3 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.State; +import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.AggregateFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; +import org.apache.iotdb.udf.api.utils.ResultValue; + +import java.nio.ByteBuffer; + +public class MyAvg implements AggregateFunction { + + static class AvgState implements State { + double sum; + long count; + + @Override + public void reset() { + sum = 0; + count = 0; + } + + @Override + public byte[] serialize() { + ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES + Long.BYTES); + buffer.putDouble(sum); + buffer.putLong(count); + + return buffer.array(); + } + + @Override + public void deserialize(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + sum = buffer.getDouble(); + count = buffer.getLong(); + } + } + + @Override + public AggregateFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() != 1) { + throw new UDFArgumentNotValidException("MyAvg only accepts one column as input"); + } + if (arguments.getDataType(0) != Type.INT32 + && arguments.getDataType(0) != Type.INT64 + && arguments.getDataType(0) != Type.FLOAT + && arguments.getDataType(0) != Type.DOUBLE) { + throw new UDFArgumentNotValidException( + "MyAvg only accepts INT32, INT64, FLOAT, DOUBLE as input"); + } + return new AggregateFunctionAnalysis.Builder() + .outputDataType(Type.DOUBLE) + .removable(true) + .build(); + } + + @Override + public State createState() { + return new AvgState(); + } + + @Override + public void addInput(State state, Record input) { + if (!input.isNull(0)) { + AvgState avgState = (AvgState) state; + switch (input.getDataType(0)) { + case INT32: + avgState.sum += input.getInt(0); + break; + case INT64: + avgState.sum += input.getLong(0); + break; + case FLOAT: + avgState.sum += input.getFloat(0); + break; + case DOUBLE: + avgState.sum += input.getDouble(0); + break; + default: + throw new UDFException("MyAvg only accepts INT32, INT64, FLOAT, DOUBLE as input"); + } + avgState.count++; + } + } + + @Override + public void combineState(State state, State rhs) { + AvgState avgState = (AvgState) state; + AvgState avgRhs = (AvgState) rhs; + avgState.sum += avgRhs.sum; + avgState.count += avgRhs.count; + } + + @Override + public void outputFinal(State state, ResultValue resultValue) { + AvgState avgState = (AvgState) state; + if (avgState.count != 0) { + resultValue.setDouble(avgState.sum / avgState.count); + } else { + resultValue.setNull(); + } + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java new file mode 100644 index 0000000000000..e1dae4ff90df8 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.State; +import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.relational.AggregateFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; +import org.apache.iotdb.udf.api.utils.ResultValue; + +import java.nio.ByteBuffer; + +public class MyCount implements AggregateFunction { + + static class CountState implements State { + long count; + + @Override + public void reset() { + count = 0; + } + + @Override + public byte[] serialize() { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(count); + + return buffer.array(); + } + + @Override + public void deserialize(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + count = buffer.getLong(); + } + } + + @Override + public AggregateFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() == 0) { + throw new UDFArgumentNotValidException("MyCount accepts at least one parameter"); + } + return new AggregateFunctionAnalysis.Builder().outputDataType(Type.INT64).build(); + } + + @Override + public State createState() { + return new CountState(); + } + + @Override + public void addInput(State state, Record input) { + CountState countState = (CountState) state; + for (int i = 0; i < input.size(); i++) { + if (!input.isNull(i)) { + countState.count++; + break; + } + } + } + + @Override + public void combineState(State state, State rhs) { + ((CountState) state).count += ((CountState) rhs).count; + } + + @Override + public void outputFinal(State state, ResultValue resultValue) { + resultValue.setLong(((CountState) state).count); + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyErrorTableFunction.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyErrorTableFunction.java new file mode 100644 index 0000000000000..1dd44653ecb43 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyErrorTableFunction.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** It is a table function with wrong implementation, which is only used for testing. */ +public class MyErrorTableFunction implements TableFunction { + + private final String TBL_PARAM = "DATA"; + private final String N_PARAM = "N"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder() + .name(TBL_PARAM) + .rowSemantics() + .passThroughColumns() + .build(), + ScalarParameterSpecification.builder().name(N_PARAM).type(Type.INT32).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + // there are different error type. + ScalarArgument n = (ScalarArgument) arguments.get("N"); + int nValue = (int) n.getValue(); + if (nValue == 0) { + // do not set required columns + return TableFunctionAnalysis.builder() + .properColumnSchema( + DescribedSchema.builder().addField("proper_column", Type.INT32).build()) + .handle(new MapTableFunctionHandle()) + .build(); + } else if (nValue == 1) { + // set empty required columns + return TableFunctionAnalysis.builder() + .properColumnSchema( + DescribedSchema.builder().addField("proper_column", Type.INT32).build()) + .requiredColumns(TBL_PARAM, Collections.emptyList()) + .handle(new MapTableFunctionHandle()) + .build(); + } else if (nValue == 2) { + // set negative required columns + return TableFunctionAnalysis.builder() + .properColumnSchema( + DescribedSchema.builder().addField("proper_column", Type.INT32).build()) + .requiredColumns(TBL_PARAM, Collections.singletonList(-1)) + .handle(new MapTableFunctionHandle()) + .build(); + } else if (nValue == 3) { + // set required columns out of bound (0~10) + return TableFunctionAnalysis.builder() + .properColumnSchema( + DescribedSchema.builder().addField("proper_column", Type.INT32).build()) + .requiredColumns(TBL_PARAM, IntStream.range(0, 11).boxed().collect(Collectors.toList())) + .handle(new MapTableFunctionHandle()) + .build(); + } else if (nValue == 4) { + // specify required columns to unknown table + return TableFunctionAnalysis.builder() + .properColumnSchema( + DescribedSchema.builder().addField("proper_column", Type.INT32).build()) + .requiredColumns("TIMECHO", Collections.singletonList(1)) + .handle(new MapTableFunctionHandle()) + .build(); + } + throw new UDFArgumentNotValidException("unexpected argument value"); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return (input, properColumnBuilders, passThroughIndexBuilder) -> { + // do nothing + }; + } + }; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyExcludeColumn.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyExcludeColumn.java new file mode 100644 index 0000000000000..49cc5a490d880 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyExcludeColumn.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class MyExcludeColumn implements TableFunction { + private final String TBL_PARAM = "DATA"; + private final String COL_PARAM = "EXCLUDE"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder().name(TBL_PARAM).rowSemantics().build(), + ScalarParameterSpecification.builder().name(COL_PARAM).type(Type.STRING).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + TableArgument tableArgument = (TableArgument) arguments.get(TBL_PARAM); + String excludeColumn = (String) ((ScalarArgument) arguments.get(COL_PARAM)).getValue(); + List requiredColumns = new ArrayList<>(); + DescribedSchema.Builder schemaBuilder = DescribedSchema.builder(); + for (int i = 0; i < tableArgument.getFieldNames().size(); i++) { + Optional fieldName = tableArgument.getFieldNames().get(i); + if (!fieldName.isPresent() || !fieldName.get().equalsIgnoreCase(excludeColumn)) { + requiredColumns.add(i); + schemaBuilder.addField(fieldName, tableArgument.getFieldTypes().get(i)); + } + } + return TableFunctionAnalysis.builder() + .properColumnSchema(schemaBuilder.build()) + .requiredColumns(TBL_PARAM, requiredColumns) + .handle(new MapTableFunctionHandle()) + .build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return (input, properColumnBuilders, passThroughIndexBuilder) -> { + for (int i = 0; i < input.size(); i++) { + if (input.isNull(i)) { + properColumnBuilders.get(i).appendNull(); + } else { + properColumnBuilders.get(i).writeObject(input.getObject(i)); + } + } + }; + } + }; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyRepeatWithIndex.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyRepeatWithIndex.java new file mode 100644 index 0000000000000..8fe032bfb5e9d --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyRepeatWithIndex.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MyRepeatWithIndex implements TableFunction { + private final String TBL_PARAM = "DATA"; + private final String N_PARAM = "N"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder() + .name(TBL_PARAM) + .rowSemantics() + .passThroughColumns() + .build(), + ScalarParameterSpecification.builder().name(N_PARAM).type(Type.INT32).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + + ScalarArgument count = (ScalarArgument) arguments.get("N"); + if (count == null) { + throw new UDFArgumentNotValidException("count argument for function repeat() is missing"); + } else if ((int) count.getValue() <= 0) { + throw new UDFArgumentNotValidException( + "count argument for function repeat() must be positive"); + } + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder().addProperty(N_PARAM, count.getValue()).build(); + return TableFunctionAnalysis.builder() + .properColumnSchema(DescribedSchema.builder().addField("repeat_index", Type.INT32).build()) + .requiredColumns( + TBL_PARAM, + Collections.singletonList(0)) // per spec, function must require at least one column + .handle(handle) + .build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return new TableFunctionDataProcessor() { + private final int n = + (int) ((MapTableFunctionHandle) tableFunctionHandle).getProperty(N_PARAM); + private long recordIndex = 0; + + @Override + public void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder) { + for (int i = 0; i < n; i++) { + properColumnBuilders.get(0).writeInt(i); + passThroughIndexBuilder.writeLong(recordIndex); + } + recordIndex++; + } + }; + } + }; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyRepeatWithoutIndex.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyRepeatWithoutIndex.java new file mode 100644 index 0000000000000..42b2e331dd325 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyRepeatWithoutIndex.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MyRepeatWithoutIndex implements TableFunction { + private final String TBL_PARAM = "DATA"; + private final String N_PARAM = "N"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder() + .name(TBL_PARAM) + .rowSemantics() + .passThroughColumns() + .build(), + ScalarParameterSpecification.builder().name(N_PARAM).type(Type.INT32).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + + ScalarArgument count = (ScalarArgument) arguments.get("N"); + if (count == null) { + throw new UDFArgumentNotValidException("count argument for function repeat() is missing"); + } else if ((int) count.getValue() <= 0) { + throw new UDFArgumentNotValidException( + "count argument for function repeat() must be positive"); + } + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder().addProperty(N_PARAM, count.getValue()).build(); + return TableFunctionAnalysis.builder() + .requiredColumns( + TBL_PARAM, + Collections.singletonList(0)) // per spec, function must require at least one column + .handle(handle) + .build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return new TableFunctionDataProcessor() { + private final int n = + (int) ((MapTableFunctionHandle) tableFunctionHandle).getProperty(N_PARAM); + private long recordIndex = 0; + + @Override + public void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder) { + for (int i = 0; i < n; i++) { + passThroughIndexBuilder.writeLong(recordIndex); + } + recordIndex++; + } + }; + } + }; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MySelectColumn.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MySelectColumn.java new file mode 100644 index 0000000000000..6cc0d683e1007 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MySelectColumn.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.EmptyTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class MySelectColumn implements TableFunction { + private final String TBL_PARAM = "DATA"; + private final String COL_PARAM = "SELECT"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder().name(TBL_PARAM).build(), + ScalarParameterSpecification.builder().name(COL_PARAM).type(Type.STRING).build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + TableArgument tableArgument = (TableArgument) arguments.get(TBL_PARAM); + String selectColumn = (String) ((ScalarArgument) arguments.get(COL_PARAM)).getValue(); + List requiredColumns = new ArrayList<>(); + DescribedSchema.Builder schemaBuilder = DescribedSchema.builder(); + for (int i = 0; i < tableArgument.getFieldNames().size(); i++) { + Optional fieldName = tableArgument.getFieldNames().get(i); + if (fieldName.isPresent() && fieldName.get().equalsIgnoreCase(selectColumn)) { + requiredColumns.add(i); + schemaBuilder.addField(fieldName, tableArgument.getFieldTypes().get(i)); + } + } + return TableFunctionAnalysis.builder() + .properColumnSchema(schemaBuilder.build()) + .requiredColumns(TBL_PARAM, requiredColumns) + .handle(new EmptyTableFunctionHandle()) + .build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new EmptyTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return (input, properColumnBuilders, passThroughIndexBuilder) -> { + for (int i = 0; i < input.size(); i++) { + if (input.isNull(i)) { + properColumnBuilders.get(i).appendNull(); + } else { + properColumnBuilders.get(i).writeObject(input.getObject(i)); + } + } + }; + } + }; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MySplit.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MySplit.java new file mode 100644 index 0000000000000..5da4f6a30bfb4 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MySplit.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.udf.example.relational; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionLeafProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.utils.Binary; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class MySplit implements TableFunction { + private final String INPUT_PARAMETER_NAME = "input"; + private final String SPLIT_PARAMETER_NAME = "split"; + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + ScalarParameterSpecification.builder().name(INPUT_PARAMETER_NAME).type(Type.STRING).build(), + ScalarParameterSpecification.builder() + .name(SPLIT_PARAMETER_NAME) + .type(Type.STRING) + .defaultValue(",") + .build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + DescribedSchema schema = DescribedSchema.builder().addField("output", Type.STRING).build(); + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder() + .addProperty( + INPUT_PARAMETER_NAME, + ((ScalarArgument) arguments.get(INPUT_PARAMETER_NAME)).getValue()) + .addProperty( + SPLIT_PARAMETER_NAME, + ((ScalarArgument) arguments.get(SPLIT_PARAMETER_NAME)).getValue()) + .build(); + return TableFunctionAnalysis.builder().properColumnSchema(schema).handle(handle).build(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionLeafProcessor getSplitProcessor() { + return new SplitProcessor( + (String) + ((MapTableFunctionHandle) tableFunctionHandle).getProperty(INPUT_PARAMETER_NAME), + (String) + ((MapTableFunctionHandle) tableFunctionHandle).getProperty(SPLIT_PARAMETER_NAME)); + } + }; + } + + private static class SplitProcessor implements TableFunctionLeafProcessor { + private final String input; + private final String split; + private boolean finish = false; + + SplitProcessor(String input, String split) { + this.input = input; + this.split = split; + } + + @Override + public void process(List columnBuilders) { + for (String s : input.split(split)) { + columnBuilders.get(0).writeBinary(new Binary(s, Charset.defaultCharset())); + } + finish = true; + } + + @Override + public boolean isFinish() { + return finish; + } + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/db/trigger/example/TriggerFireTimesCounter.java b/integration-test/src/main/java/org/apache/iotdb/db/trigger/example/TriggerFireTimesCounter.java index 2ddfbb171257e..fd682a03a2994 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/trigger/example/TriggerFireTimesCounter.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/trigger/example/TriggerFireTimesCounter.java @@ -86,7 +86,7 @@ public boolean fire(Tablet tablet) throws Exception { Thread.sleep(100); } } - int rows = tablet.rowSize; + int rows = tablet.getRowSize(); if (fileLock != null && fileLock.isValid()) { String records = System.lineSeparator() + rows; ByteBuffer byteBuffer = ByteBuffer.allocate(1024); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java index 8ea62903e1f41..a71e4e7a3aeea 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java @@ -19,6 +19,8 @@ package org.apache.iotdb.it.env; +import org.apache.iotdb.it.env.cluster.env.AIEnv; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.env.Cluster1Env; import org.apache.iotdb.it.env.cluster.env.SimpleEnv; import org.apache.iotdb.it.env.remote.env.RemoteServerEnv; @@ -44,14 +46,19 @@ public static BaseEnv getEnv() { EnvType envType = EnvType.getSystemEnvType(); switch (envType) { case Simple: + case TABLE_SIMPLE: env = new SimpleEnv(); break; case Cluster1: + case TABLE_CLUSTER1: env = new Cluster1Env(); break; case Remote: env = new RemoteServerEnv(); break; + case AI: + env = new AIEnv(); + break; case MultiCluster: logger.warn( "EnvFactory only supports EnvType Simple, Cluster1 and Remote, please use MultiEnvFactory instead."); @@ -69,4 +76,8 @@ public static BaseEnv getEnv() { } return env; } + + public static AbstractEnv getAbstractEnv() { + return (AbstractEnv) getEnv(); + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java index 5be5da287c741..7c2ee415cf1b4 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvType.java @@ -24,7 +24,9 @@ public enum EnvType { Simple, Cluster1, MultiCluster, - ; + AI, + TABLE_SIMPLE, + TABLE_CLUSTER1; public static EnvType getSystemEnvType() { String envValue = System.getProperty("TestEnv", Simple.name()); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/MultiEnvFactory.java b/integration-test/src/main/java/org/apache/iotdb/it/env/MultiEnvFactory.java index f8c28567be1cb..5832f1c485bc5 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/MultiEnvFactory.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/MultiEnvFactory.java @@ -38,24 +38,25 @@ private MultiEnvFactory() { // Empty constructor } - public static void setTestMethodName(String testMethodName) { + public static void setTestMethodName(final String testMethodName) { currentMethodName = testMethodName; + envList.forEach(baseEnv -> baseEnv.setTestMethodName(testMethodName)); } /** Get an environment with the specific index. */ - public static BaseEnv getEnv(int index) throws IndexOutOfBoundsException { + public static BaseEnv getEnv(final int index) throws IndexOutOfBoundsException { return envList.get(index); } /** Create several environments according to the specific number. */ - public static void createEnv(int num) { + public static void createEnv(final int num) { // Not judge EnvType for individual test convenience - long startTime = System.currentTimeMillis(); + final long startTime = System.currentTimeMillis(); for (int i = 0; i < num; ++i) { try { Class.forName(Config.JDBC_DRIVER_NAME); envList.add(new MultiClusterEnv(startTime, i, currentMethodName)); - } catch (ClassNotFoundException e) { + } catch (final ClassNotFoundException e) { logger.error("Create env error", e); System.exit(-1); } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index 12af0d025f75c..3162f47ebe8bf 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -43,6 +43,7 @@ public class ClusterConstant { public static final String DEFAULT_DATA_NODE_PROPERTIES = "DefaultDataNodeProperties"; public static final String DEFAULT_DATA_NODE_COMMON_PROPERTIES = "DefaultDataNodeCommonProperties"; + public static final String DATA_REGION_PER_DATANODE = "integrationTest.dataRegionPerDataNode"; // Cluster Configurations public static final String CLUSTER_CONFIGURATIONS = "ClusterConfigurations"; @@ -52,6 +53,8 @@ public class ClusterConstant { public static final String SCALABLE_SINGLE_NODE_MODE = "ScalableSingleNodeMode"; public static final String HIGH_PERFORMANCE_MODE = "HighPerformanceMode"; public static final String STRONG_CONSISTENCY_CLUSTER_MODE = "StrongConsistencyClusterMode"; + public static final String PIPE_CONSENSUS_BATCH_MODE = "PipeConsensusBatchMode"; + public static final String PIPE_CONSENSUS_STREAM_MODE = "PipeConsensusStreamMode"; // System arguments in pom.xml public static final String LIGHT_WEIGHT_STANDALONE_MODE_CONFIG_NODE_NUM = @@ -114,10 +117,37 @@ public class ClusterConstant { public static final String STRONG_CONSISTENCY_CLUSTER_MODE_DATA_REGION_REPLICA_NUM = "strongConsistencyClusterMode.dataRegionReplicaNumber"; + public static final String PIPE_CONSENSUS_BATCH_MODE_CONFIG_NODE_NUM = + "pipeConsensusBatchMode.configNodeNumber"; + public static final String PIPE_CONSENSUS_BATCH_MODE_DATA_NODE_NUM = + "pipeConsensusBatchMode.dataNodeNumber"; + public static final String PIPE_CONSENSUS_BATCH_MODE_CONFIG_NODE_CONSENSUS = + "pipeConsensusBatchMode.configNodeConsensus"; + public static final String PIPE_CONSENSUS_BATCH_MODE_SCHEMA_REGION_CONSENSUS = + "pipeConsensusBatchMode.schemaRegionConsensus"; + public static final String PIPE_CONSENSUS_BATCH_MODE_DATA_REGION_CONSENSUS = + "pipeConsensusBatchMode.dataRegionConsensus"; + public static final String PIPE_CONSENSUS_BATCH_MODE_SCHEMA_REGION_REPLICA_NUM = + "pipeConsensusBatchMode.schemaRegionReplicaNumber"; + public static final String PIPE_CONSENSUS_BATCH_MODE_DATA_REGION_REPLICA_NUM = + "pipeConsensusBatchMode.dataRegionReplicaNumber"; + + public static final String PIPE_CONSENSUS_STREAM_MODE_CONFIG_NODE_NUM = + "pipeConsensusStreamMode.configNodeNumber"; + public static final String PIPE_CONSENSUS_STREAM_MODE_DATA_NODE_NUM = + "pipeConsensusStreamMode.dataNodeNumber"; + public static final String PIPE_CONSENSUS_STREAM_MODE_CONFIG_NODE_CONSENSUS = + "pipeConsensusStreamMode.configNodeConsensus"; + public static final String PIPE_CONSENSUS_STREAM_MODE_SCHEMA_REGION_CONSENSUS = + "pipeConsensusStreamMode.schemaRegionConsensus"; + public static final String PIPE_CONSENSUS_STREAM_MODE_DATA_REGION_CONSENSUS = + "pipeConsensusStreamMode.dataRegionConsensus"; + public static final String PIPE_CONSENSUS_STREAM_MODE_SCHEMA_REGION_REPLICA_NUM = + "pipeConsensusStreamMode.schemaRegionReplicaNumber"; + public static final String PIPE_CONSENSUS_STREAM_MODE_DATA_REGION_REPLICA_NUM = + "pipeConsensusStreamMode.dataRegionReplicaNumber"; + // Property file names - public static final String CONFIG_NODE_PROPERTIES_FILE = "iotdb-confignode.properties"; - public static final String DATA_NODE_PROPERTIES_FILE = "iotdb-datanode.properties"; - public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties"; public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties"; // Properties' keys @@ -128,11 +158,13 @@ public class ClusterConstant { "schema_region_consensus_protocol_class"; public static final String DATA_REGION_CONSENSUS_PROTOCOL_CLASS = "data_region_consensus_protocol_class"; + public static final String IOT_CONSENSUS_V2_MODE = "iot_consensus_v2_mode"; public static final String SCHEMA_REPLICATION_FACTOR = "schema_replication_factor"; public static final String DATA_REPLICATION_FACTOR = "data_replication_factor"; public static final String MQTT_HOST = "mqtt_host"; public static final String MQTT_PORT = "mqtt_port"; + public static final String MQTT_DATA_PATH = "mqtt_data_path"; public static final String UDF_LIB_DIR = "udf_lib_dir"; public static final String TRIGGER_LIB_DIR = "trigger_lib_dir"; public static final String PIPE_LIB_DIR = "pipe_lib_dir"; @@ -142,10 +174,7 @@ public class ClusterConstant { // ConfigNode public static final String CN_SYSTEM_DIR = "cn_system_dir"; public static final String CN_CONSENSUS_DIR = "cn_consensus_dir"; - public static final String CN_METRIC_PROMETHEUS_REPORTER_PORT = - "cn_metric_prometheus_reporter_port"; public static final String CN_METRIC_IOTDB_REPORTER_HOST = "cn_metric_iotdb_reporter_host"; - public static final String CN_METRIC_IOTDB_REPORTER_PORT = "cn_metric_iotdb_reporter_port"; public static final String CN_CONNECTION_TIMEOUT_MS = "cn_connection_timeout_ms"; @@ -157,13 +186,10 @@ public class ClusterConstant { public static final String DN_TRACING_DIR = "dn_tracing_dir"; public static final String DN_SYNC_DIR = "dn_sync_dir"; public static final String DN_METRIC_IOTDB_REPORTER_HOST = "dn_metric_iotdb_reporter_host"; - public static final String DN_METRIC_PROMETHEUS_REPORTER_PORT = - "dn_metric_prometheus_reporter_port"; public static final String DN_MPP_DATA_EXCHANGE_PORT = "dn_mpp_data_exchange_port"; public static final String DN_DATA_REGION_CONSENSUS_PORT = "dn_data_region_consensus_port"; public static final String DN_SCHEMA_REGION_CONSENSUS_PORT = "dn_schema_region_consensus_port"; - public static final String PIPE_AIR_GAP_RECEIVER_ENABLED = "pipe_air_gap_receiver_enabled"; public static final String PIPE_AIR_GAP_RECEIVER_PORT = "pipe_air_gap_receiver_port"; public static final String MAX_TSBLOCK_SIZE_IN_BYTES = "max_tsblock_size_in_bytes"; public static final String PAGE_SIZE_IN_BYTE = "page_size_in_byte"; @@ -171,13 +197,24 @@ public class ClusterConstant { "dn_join_cluster_retry_interval_ms"; public static final String DN_CONNECTION_TIMEOUT_MS = "dn_connection_timeout_ms"; public static final String DN_METRIC_INTERNAL_REPORTER_TYPE = "dn_metric_internal_reporter_type"; + public static final String CONFIG_NODE_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX = + "config_node_ratis_log_appender_buffer_size_max"; + public static final String WAL_BUFFER_SIZE_IN_BYTE = "wal_buffer_size_in_byte"; + public static final String SCHEMA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX = + "schema_region_ratis_log_appender_buffer_size_max"; + public static final String DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX = + "data_region_ratis_log_appender_buffer_size_max"; + public static final String DATA_REGION_PER_DATA_NODE = "data_region_per_data_node"; // Paths public static final String USER_DIR = "user.dir"; public static final String TARGET = "target"; + public static final String PYTHON_PATH = "venv/bin/python3"; public static final String DATA_NODE_NAME = "DataNode"; + public static final String AI_NODE_NAME = "AINode"; + public static final String LOCK_FILE_PATH = System.getProperty(USER_DIR) + File.separator + TARGET + File.separator + "lock-"; public static final String TEMPLATE_NODE_PATH = @@ -195,7 +232,6 @@ public class ClusterConstant { // Env Constant public static final int NODE_START_TIMEOUT = 100; - public static final int PROBE_TIMEOUT_MS = 2000; public static final int NODE_NETWORK_TIMEOUT_MS = 0; public static final String ZERO_TIME_ZONE = "GMT+0"; @@ -207,9 +243,7 @@ public class ClusterConstant { public static final String SIMPLE_CONSENSUS_STR = "Simple"; public static final String RATIS_CONSENSUS_STR = "Ratis"; public static final String IOT_CONSENSUS_STR = "IoT"; - public static final String PIPE_CONSENSUS_STR = "Pipe"; - public static final String STREAM_CONSENSUS_STR = "Stream"; - public static final String BATCH_CONSENSUS_STR = "Batch"; + public static final String IOT_CONSENSUS_V2_STR = "IoTV2"; public static final String JAVA_CMD = System.getProperty("java.home") diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index f3c9527e5952f..f9315ac787f9e 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -30,14 +30,11 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.iotdb.consensus.ConsensusFactory.FAST_IOT_CONSENSUS; import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS; import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS_V2; import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS; -import static org.apache.iotdb.consensus.ConsensusFactory.REAL_PIPE_CONSENSUS; import static org.apache.iotdb.consensus.ConsensusFactory.SIMPLE_CONSENSUS; import static org.apache.iotdb.db.utils.DateTimeUtils.convertLongToDate; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.BATCH_CONSENSUS_STR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CLUSTER_CONFIGURATIONS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_NUM; @@ -47,17 +44,22 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.HIGH_PERFORMANCE_MODE_CONFIG_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.HIGH_PERFORMANCE_MODE_DATA_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOT_CONSENSUS_STR; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOT_CONSENSUS_V2_STR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.LIGHT_WEIGHT_STANDALONE_MODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.LIGHT_WEIGHT_STANDALONE_MODE_CONFIG_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.LIGHT_WEIGHT_STANDALONE_MODE_DATA_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.LOCK_FILE_PATH; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STR; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_CONFIG_NODE_NUM; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_DATA_NODE_NUM; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_CONFIG_NODE_NUM; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_DATA_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.RATIS_CONSENSUS_STR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCALABLE_SINGLE_NODE_MODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCALABLE_SINGLE_NODE_MODE_CONFIG_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCALABLE_SINGLE_NODE_MODE_DATA_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SIMPLE_CONSENSUS_STR; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.STREAM_CONSENSUS_STR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.STRONG_CONSISTENCY_CLUSTER_MODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.STRONG_CONSISTENCY_CLUSTER_MODE_CONFIG_NODE_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.STRONG_CONSISTENCY_CLUSTER_MODE_DATA_NODE_NUM; @@ -70,22 +72,22 @@ public static int[] searchAvailablePorts() { while (true) { int randomPortStart = 1000 + (int) (Math.random() * (1999 - 1000)); randomPortStart = randomPortStart * (length + 1) + 1; - String lockFilePath = getLockFilePath(randomPortStart); - File lockFile = new File(lockFilePath); + final String lockFilePath = getLockFilePath(randomPortStart); + final File lockFile = new File(lockFilePath); try { // Lock the ports first to avoid to be occupied by other ForkedBooters during ports // available detecting if (!lockFile.createNewFile()) { continue; } - List requiredPorts = + final List requiredPorts = IntStream.rangeClosed(randomPortStart, randomPortStart + length) .boxed() .collect(Collectors.toList()); if (checkPortsAvailable(requiredPorts)) { return requiredPorts.stream().mapToInt(Integer::intValue).toArray(); } - } catch (IOException e) { + } catch (final IOException ignore) { // ignore } // Delete the lock file if the ports can't be used or some error happens @@ -95,39 +97,35 @@ public static int[] searchAvailablePorts() { } } - private static boolean checkPortsAvailable(List ports) { - String cmd = getSearchAvailablePortCmd(ports); + private static boolean checkPortsAvailable(final List ports) { + final String cmd = getSearchAvailablePortCmd(ports); try { - Process proc = Runtime.getRuntime().exec(cmd); - return proc.waitFor() == 1; - } catch (IOException e) { + return Runtime.getRuntime().exec(cmd).waitFor() == 1; + } catch (final IOException ignore) { // ignore - } catch (InterruptedException e) { + } catch (final InterruptedException e) { Thread.currentThread().interrupt(); } return false; } - private static String getSearchAvailablePortCmd(List ports) { - if (SystemUtils.IS_OS_WINDOWS) { - return getWindowsSearchPortCmd(ports); - } - return getUnixSearchPortCmd(ports); + private static String getSearchAvailablePortCmd(final List ports) { + return SystemUtils.IS_OS_WINDOWS ? getWindowsSearchPortCmd(ports) : getUnixSearchPortCmd(ports); } - private static String getWindowsSearchPortCmd(List ports) { - String cmd = "netstat -aon -p tcp | findStr "; - return cmd + private static String getWindowsSearchPortCmd(final List ports) { + return "netstat -aon -p tcp | findStr " + ports.stream().map(v -> "/C:'127.0.0.1:" + v + "'").collect(Collectors.joining(" ")); } - private static String getUnixSearchPortCmd(List ports) { - String cmd = "lsof -iTCP -sTCP:LISTEN -P -n | awk '{print $9}' | grep -E "; - return cmd + ports.stream().map(String::valueOf).collect(Collectors.joining("|")) + "\""; + private static String getUnixSearchPortCmd(final List ports) { + return "lsof -iTCP -sTCP:LISTEN -P -n | awk '{print $9}' | grep -E " + + ports.stream().map(String::valueOf).collect(Collectors.joining("|")) + + "\""; } - private static Pair getClusterNodesNum(int index) { - String valueStr = System.getProperty(CLUSTER_CONFIGURATIONS); + private static Pair getClusterNodesNum(final int index) { + final String valueStr = System.getProperty(CLUSTER_CONFIGURATIONS); if (valueStr == null) { return null; } @@ -150,21 +148,29 @@ private static Pair getClusterNodesNum(int index) { return new Pair<>( Integer.parseInt(System.getProperty(STRONG_CONSISTENCY_CLUSTER_MODE_CONFIG_NODE_NUM)), Integer.parseInt(System.getProperty(STRONG_CONSISTENCY_CLUSTER_MODE_DATA_NODE_NUM))); + case PIPE_CONSENSUS_BATCH_MODE: + return new Pair<>( + Integer.parseInt(System.getProperty(PIPE_CONSENSUS_BATCH_MODE_CONFIG_NODE_NUM)), + Integer.parseInt(System.getProperty(PIPE_CONSENSUS_BATCH_MODE_DATA_NODE_NUM))); + case PIPE_CONSENSUS_STREAM_MODE: + return new Pair<>( + Integer.parseInt(System.getProperty(PIPE_CONSENSUS_STREAM_MODE_CONFIG_NODE_NUM)), + Integer.parseInt(System.getProperty(PIPE_CONSENSUS_STREAM_MODE_DATA_NODE_NUM))); default: // Print nothing to avoid polluting test outputs return null; } - } catch (NumberFormatException ignore) { + } catch (final NumberFormatException ignore) { return null; } } - public static String getLockFilePath(int port) { + public static String getLockFilePath(final int port) { return LOCK_FILE_PATH + port; } public static Pair getNodeNum() { - Pair nodesNum = getClusterNodesNum(0); + final Pair nodesNum = getClusterNodesNum(0); if (nodesNum != null) { return nodesNum; } @@ -173,8 +179,8 @@ public static Pair getNodeNum() { getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 3, 0)); } - public static Pair getNodeNum(int index) { - Pair nodesNum = getClusterNodesNum(index); + public static Pair getNodeNum(final int index) { + final Pair nodesNum = getClusterNodesNum(index); if (nodesNum != null) { return nodesNum; } @@ -183,38 +189,38 @@ public static Pair getNodeNum(int index) { getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 3, index)); } - public static String getFilePathFromSysVar(String key, int index) { - String valueStr = System.getProperty(key); + public static String getFilePathFromSysVar(final String key, final int index) { + final String valueStr = System.getProperty(key); if (valueStr == null) { return null; } return System.getProperty(USER_DIR) + getValueOfIndex(valueStr, index); } - public static int getIntFromSysVar(String key, int defaultValue, int index) { - String valueStr = System.getProperty(key); + public static int getIntFromSysVar(final String key, final int defaultValue, final int index) { + final String valueStr = System.getProperty(key); if (valueStr == null) { return defaultValue; } - String value = getValueOfIndex(valueStr, index); + final String value = getValueOfIndex(valueStr, index); try { return Integer.parseInt(value); - } catch (NumberFormatException e) { + } catch (final NumberFormatException e) { throw new IllegalArgumentException("Invalid property value: " + value + " of key " + key); } } - public static String getValueOfIndex(String valueStr, int index) { - String[] values = valueStr.split(DELIMITER); + public static String getValueOfIndex(final String valueStr, final int index) { + final String[] values = valueStr.split(DELIMITER); return index <= values.length - 1 ? values[index] : values[values.length - 1]; } - public static String getTimeForLogDirectory(long startTime) { + public static String getTimeForLogDirectory(final long startTime) { return convertLongToDate(startTime, "ms").replace(":", DIR_TIME_REPLACEMENT); } - public static String fromConsensusFullNameToAbbr(String consensus) { + public static String fromConsensusFullNameToAbbr(final String consensus) { switch (consensus) { case SIMPLE_CONSENSUS: return SIMPLE_CONSENSUS_STR; @@ -222,18 +228,14 @@ public static String fromConsensusFullNameToAbbr(String consensus) { return RATIS_CONSENSUS_STR; case IOT_CONSENSUS: return IOT_CONSENSUS_STR; - case REAL_PIPE_CONSENSUS: - return PIPE_CONSENSUS_STR; case IOT_CONSENSUS_V2: - return STREAM_CONSENSUS_STR; - case FAST_IOT_CONSENSUS: - return BATCH_CONSENSUS_STR; + return IOT_CONSENSUS_V2_STR; default: throw new IllegalArgumentException("Unknown consensus type: " + consensus); } } - public static String fromConsensusAbbrToFullName(String consensus) { + public static String fromConsensusAbbrToFullName(final String consensus) { switch (consensus) { case SIMPLE_CONSENSUS_STR: return SIMPLE_CONSENSUS; @@ -241,12 +243,8 @@ public static String fromConsensusAbbrToFullName(String consensus) { return RATIS_CONSENSUS; case IOT_CONSENSUS_STR: return IOT_CONSENSUS; - case PIPE_CONSENSUS_STR: - return REAL_PIPE_CONSENSUS; - case STREAM_CONSENSUS_STR: + case IOT_CONSENSUS_V2_STR: return IOT_CONSENSUS_V2; - case BATCH_CONSENSUS_STR: - return FAST_IOT_CONSENSUS; default: throw new IllegalArgumentException("Unknown consensus type: " + consensus); } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 9a5e066d6419b..4edfab671869d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -97,6 +97,18 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptType(String encryptType) { + setProperty("encrypt_type", encryptType); + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + setProperty("encrypt_key_path", encryptKeyPath); + return this; + } + @Override public CommonConfig setUdfMemoryBudgetInMB(float udfCollectorMemoryBudgetInMB) { // udf_memory_budget_in_mb @@ -124,11 +136,9 @@ public CommonConfig setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompac } @Override - public CommonConfig setMaxInnerCompactionCandidateFileNum( - int maxInnerCompactionCandidateFileNum) { + public CommonConfig setInnerCompactionCandidateFileNum(int maxInnerCompactionCandidateFileNum) { setProperty( - "max_inner_compaction_candidate_file_num", - String.valueOf(maxInnerCompactionCandidateFileNum)); + "inner_compaction_candidate_file_num", String.valueOf(maxInnerCompactionCandidateFileNum)); return this; } @@ -151,8 +161,8 @@ public CommonConfig setPrimitiveArraySize(int primitiveArraySize) { } @Override - public CommonConfig setAvgSeriesPointNumberThreshold(int avgSeriesPointNumberThreshold) { - setProperty("avg_series_point_number_threshold", String.valueOf(avgSeriesPointNumberThreshold)); + public CommonConfig setTargetChunkPointNum(int targetChunkPointNum) { + setProperty("target_chunk_point_num", String.valueOf(targetChunkPointNum)); return this; } @@ -187,6 +197,12 @@ public CommonConfig setDataRegionConsensusProtocolClass(String dataRegionConsens return this; } + @Override + public CommonConfig setIoTConsensusV2Mode(String ioTConsensusV2Mode) { + setProperty("iot_consensus_v2_mode", ioTConsensusV2Mode); + return this; + } + @Override public CommonConfig setSchemaRegionGroupExtensionPolicy(String schemaRegionGroupExtensionPolicy) { setProperty("schema_region_group_extension_policy", schemaRegionGroupExtensionPolicy); @@ -232,6 +248,12 @@ public CommonConfig setTimePartitionInterval(long timePartitionInterval) { return this; } + @Override + public CommonConfig setTTLCheckInterval(long ttlCheckInterval) { + setProperty("ttl_check_interval", String.valueOf(ttlCheckInterval)); + return this; + } + @Override public CommonConfig setTimePartitionOrigin(long timePartitionOrigin) { setProperty("time_partition_origin", String.valueOf(timePartitionOrigin)); @@ -293,6 +315,12 @@ public CommonConfig setEnableMQTTService(boolean enableMQTTService) { return this; } + @Override + public CommonConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { + setProperty("mqtt_payload_formatter", String.valueOf(mqttPayloadFormatter)); + return this; + } + @Override public CommonConfig setSchemaEngineMode(String schemaEngineMode) { setProperty("schema_engine_mode", schemaEngineMode); @@ -359,6 +387,12 @@ public CommonConfig setSeriesSlotNum(int seriesSlotNum) { return this; } + @Override + public CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass) { + setProperty("series_partition_executor_class", seriesPartitionExecutorClass); + return this; + } + @Override public CommonConfig setSchemaMemoryAllocate(String schemaMemoryAllocate) { setProperty("schema_memory_proportion", String.valueOf(schemaMemoryAllocate)); @@ -445,8 +479,8 @@ public CommonConfig setTagAttributeTotalSize(int tagAttributeTotalSize) { } @Override - public CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs) { - setProperty("cn_connection_timeout_ms", String.valueOf(connectionTimeoutMs)); + public CommonConfig setDnConnectionTimeoutMs(int connectionTimeoutMs) { + setProperty("dn_connection_timeout_ms", String.valueOf(connectionTimeoutMs)); return this; } @@ -476,6 +510,28 @@ public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSync return this; } + @Override + public CommonConfig setPipeConnectorRequestSliceThresholdBytes( + int pipeConnectorRequestSliceThresholdBytes) { + setProperty( + "pipe_connector_request_slice_threshold_bytes", + String.valueOf(pipeConnectorRequestSliceThresholdBytes)); + + return this; + } + + @Override + public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { + setProperty("chunk_timeseriesmeta_free_memory_proportion", queryMemoryProportion); + return this; + } + + @Override + public CommonConfig setDefaultStorageGroupLevel(int defaultStorageGroupLevel) { + setProperty("default_storage_group_level", String.valueOf(defaultStorageGroupLevel)); + return this; + } + // For part of the log directory public String getClusterConfigStr() { return fromConsensusFullNameToAbbr(properties.getProperty(CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS)) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java index 62ccbb0aa4ff6..8e4a6def36570 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java @@ -55,4 +55,16 @@ public ConfigNodeConfig setMetricReporterType(List metricReporterTypes) properties.setProperty("cn_metric_reporter_list", String.join(",", metricReporterTypes)); return this; } + + @Override + public ConfigNodeConfig setMetricPrometheusReporterUsername(String username) { + properties.setProperty("metric_prometheus_reporter_username", username); + return this; + } + + @Override + public ConfigNodeConfig setMetricPrometheusReporterPassword(String password) { + properties.setProperty("metric_prometheus_reporter_password", password); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index 0f22e0d428699..5f51e486dd8b1 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -56,6 +56,18 @@ public DataNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + @Override + public DataNodeConfig setMetricPrometheusReporterUsername(String username) { + properties.setProperty("metric_prometheus_reporter_username", username); + return this; + } + + @Override + public DataNodeConfig setMetricPrometheusReporterPassword(String password) { + properties.setProperty("metric_prometheus_reporter_password", password); + return this; + } + @Override public DataNodeConfig setEnableRestService(boolean enableRestService) { properties.setProperty("enable_rest_service", String.valueOf(enableRestService)); @@ -76,4 +88,23 @@ public DataNodeConfig setLoadTsFileAnalyzeSchemaMemorySizeInBytes( String.valueOf(loadTsFileAnalyzeSchemaMemorySizeInBytes)); return this; } + + @Override + public DataNodeConfig setCompactionScheduleInterval(long compactionScheduleInterval) { + properties.setProperty( + "compaction_schedule_interval_in_ms", String.valueOf(compactionScheduleInterval)); + return this; + } + + @Override + public DataNodeConfig setEnableMQTTService(boolean enableMQTTService) { + setProperty("enable_mqtt_service", String.valueOf(enableMQTTService)); + return this; + } + + @Override + public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { + setProperty("mqtt_payload_formatter", String.valueOf(mqttPayloadFormatter)); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 2d93a2e0dafc0..e08b752749a28 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -75,6 +75,20 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptType(String encryptType) { + cnConfig.setProperty("encrypt_type", encryptType); + dnConfig.setProperty("encrypt_type", encryptType); + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + cnConfig.setProperty("encrypt_key_path", encryptKeyPath); + dnConfig.setProperty("encrypt_key_path", encryptKeyPath); + return this; + } + @Override public CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs) { cnConfig.setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(maxMs); @@ -118,10 +132,9 @@ public CommonConfig setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompac } @Override - public CommonConfig setMaxInnerCompactionCandidateFileNum( - int maxInnerCompactionCandidateFileNum) { - cnConfig.setMaxInnerCompactionCandidateFileNum(maxInnerCompactionCandidateFileNum); - dnConfig.setMaxInnerCompactionCandidateFileNum(maxInnerCompactionCandidateFileNum); + public CommonConfig setInnerCompactionCandidateFileNum(int maxInnerCompactionCandidateFileNum) { + cnConfig.setInnerCompactionCandidateFileNum(maxInnerCompactionCandidateFileNum); + dnConfig.setInnerCompactionCandidateFileNum(maxInnerCompactionCandidateFileNum); return this; } @@ -147,9 +160,9 @@ public CommonConfig setPrimitiveArraySize(int primitiveArraySize) { } @Override - public CommonConfig setAvgSeriesPointNumberThreshold(int avgSeriesPointNumberThreshold) { - cnConfig.setAvgSeriesPointNumberThreshold(avgSeriesPointNumberThreshold); - dnConfig.setAvgSeriesPointNumberThreshold(avgSeriesPointNumberThreshold); + public CommonConfig setTargetChunkPointNum(int targetChunkPointNum) { + cnConfig.setTargetChunkPointNum(targetChunkPointNum); + dnConfig.setTargetChunkPointNum(targetChunkPointNum); return this; } @@ -182,6 +195,13 @@ public CommonConfig setDataRegionConsensusProtocolClass(String dataRegionConsens return this; } + @Override + public CommonConfig setIoTConsensusV2Mode(String ioTConsensusV2Mode) { + cnConfig.setIoTConsensusV2Mode(ioTConsensusV2Mode); + dnConfig.setIoTConsensusV2Mode(ioTConsensusV2Mode); + return this; + } + @Override public CommonConfig setSchemaRegionGroupExtensionPolicy(String schemaRegionGroupExtensionPolicy) { cnConfig.setSchemaRegionGroupExtensionPolicy(schemaRegionGroupExtensionPolicy); @@ -231,6 +251,13 @@ public CommonConfig setTimePartitionInterval(long timePartitionInterval) { return this; } + @Override + public CommonConfig setTTLCheckInterval(long ttlCheckInterval) { + cnConfig.setTTLCheckInterval(ttlCheckInterval); + dnConfig.setTTLCheckInterval(ttlCheckInterval); + return this; + } + @Override public CommonConfig setTimePartitionOrigin(long timePartitionOrigin) { cnConfig.setTimePartitionOrigin(timePartitionOrigin); @@ -289,6 +316,13 @@ public CommonConfig setEnableMQTTService(boolean enableMQTTService) { return this; } + @Override + public CommonConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { + cnConfig.setMqttPayloadFormatter(mqttPayloadFormatter); + dnConfig.setMqttPayloadFormatter(mqttPayloadFormatter); + return this; + } + @Override public CommonConfig setSchemaEngineMode(String schemaEngineMode) { cnConfig.setSchemaEngineMode(schemaEngineMode); @@ -355,6 +389,13 @@ public CommonConfig setSeriesSlotNum(int seriesSlotNum) { return this; } + @Override + public CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass) { + cnConfig.setSeriesPartitionExecutorClass(seriesPartitionExecutorClass); + dnConfig.setSeriesPartitionExecutorClass(seriesPartitionExecutorClass); + return this; + } + @Override public CommonConfig setSchemaMemoryAllocate(String schemaMemoryAllocate) { dnConfig.setSchemaMemoryAllocate(schemaMemoryAllocate); @@ -454,9 +495,9 @@ public CommonConfig setTagAttributeTotalSize(int tagAttributeTotalSize) { } @Override - public CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs) { - dnConfig.setCnConnectionTimeoutMs(connectionTimeoutMs); - cnConfig.setCnConnectionTimeoutMs(connectionTimeoutMs); + public CommonConfig setDnConnectionTimeoutMs(int connectionTimeoutMs) { + dnConfig.setDnConnectionTimeoutMs(connectionTimeoutMs); + cnConfig.setDnConnectionTimeoutMs(connectionTimeoutMs); return this; } @@ -484,4 +525,26 @@ public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSync cnConfig.setPipeMetaSyncerSyncIntervalMinutes(pipeMetaSyncerSyncIntervalMinutes); return this; } + + @Override + public CommonConfig setPipeConnectorRequestSliceThresholdBytes( + int pipeConnectorRequestSliceThresholdBytes) { + dnConfig.setPipeConnectorRequestSliceThresholdBytes(pipeConnectorRequestSliceThresholdBytes); + cnConfig.setPipeConnectorRequestSliceThresholdBytes(pipeConnectorRequestSliceThresholdBytes); + return this; + } + + @Override + public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { + dnConfig.setQueryMemoryProportion(queryMemoryProportion); + cnConfig.setQueryMemoryProportion(queryMemoryProportion); + return this; + } + + @Override + public CommonConfig setDefaultStorageGroupLevel(int defaultStorageGroupLevel) { + dnConfig.setDefaultStorageGroupLevel(defaultStorageGroupLevel); + cnConfig.setDefaultStorageGroupLevel(defaultStorageGroupLevel); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java new file mode 100644 index 0000000000000..916a2ee8cf6cf --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.it.env.cluster.env; + +public class AIEnv extends AbstractEnv { + @Override + public void initClusterEnvironment() { + initClusterEnvironment(1, 1); + } + + @Override + public void initClusterEnvironment(int configNodesNum, int dataNodesNum) { + super.initEnvironment(configNodesNum, dataNodesNum, 1000, true); + } + + @Override + public void initClusterEnvironment( + int configNodesNum, int dataNodesNum, int testWorkingRetryCount) { + super.initEnvironment(configNodesNum, dataNodesNum, testWorkingRetryCount, true); + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index 54fedec8de0f6..c64159f3169aa 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -19,9 +19,7 @@ package org.apache.iotdb.it.env.cluster.env; -import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; -import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; @@ -36,11 +34,14 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.pool.ISessionPool; +import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.config.*; +import org.apache.iotdb.it.env.cluster.node.AINodeWrapper; import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; @@ -55,9 +56,12 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.TableSessionBuilder; import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import java.io.File; @@ -65,6 +69,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.time.ZoneId; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -80,6 +85,7 @@ public abstract class AbstractEnv implements BaseEnv { private final Random rand = new Random(); protected List configNodeWrapperList = Collections.emptyList(); protected List dataNodeWrapperList = Collections.emptyList(); + protected List aiNodeWrapperList = Collections.emptyList(); protected String testMethodName = null; protected int index = 0; protected long startTime; @@ -101,7 +107,7 @@ protected AbstractEnv() { } // For multiple environment ITs, time must be consistent across environments. - protected AbstractEnv(long startTime) { + protected AbstractEnv(final long startTime) { this.startTime = startTime; this.clusterConfig = new MppClusterConfig(); } @@ -112,38 +118,49 @@ public ClusterConfig getConfig() { } @Override - public List getMetricPrometheusReporterContents() { - List result = new ArrayList<>(); + public List getMetricPrometheusReporterContents(String authHeader) { + final List result = new ArrayList<>(); // get all report content of confignodes - for (ConfigNodeWrapper configNode : this.configNodeWrapperList) { - String configNodeMetricContent = + for (final ConfigNodeWrapper configNode : this.configNodeWrapperList) { + final String configNodeMetricContent = getUrlContent( Config.IOTDB_HTTP_URL_PREFIX + configNode.getIp() + ":" + configNode.getMetricPort() - + "/metrics"); + + "/metrics", + authHeader); result.add(configNodeMetricContent); } // get all report content of datanodes - for (DataNodeWrapper dataNode : this.dataNodeWrapperList) { - String dataNodeMetricContent = + for (final DataNodeWrapper dataNode : this.dataNodeWrapperList) { + final String dataNodeMetricContent = getUrlContent( Config.IOTDB_HTTP_URL_PREFIX + dataNode.getIp() + ":" + dataNode.getMetricPort() - + "/metrics"); + + "/metrics", + authHeader); result.add(dataNodeMetricContent); } return result; } - protected void initEnvironment(int configNodesNum, int dataNodesNum) { + protected void initEnvironment(final int configNodesNum, final int dataNodesNum) { initEnvironment(configNodesNum, dataNodesNum, retryCount); } - protected void initEnvironment(int configNodesNum, int dataNodesNum, int retryCount) { + protected void initEnvironment( + final int configNodesNum, final int dataNodesNum, final int testWorkingRetryCount) { + initEnvironment(configNodesNum, dataNodesNum, testWorkingRetryCount, false); + } + + protected void initEnvironment( + final int configNodesNum, + final int dataNodesNum, + final int retryCount, + final boolean addAINode) { this.retryCount = retryCount; this.configNodeWrapperList = new ArrayList<>(); this.dataNodeWrapperList = new ArrayList<>(); @@ -154,7 +171,7 @@ protected void initEnvironment(int configNodesNum, int dataNodesNum, int retryCo final String testClassName = getTestClassName(); - ConfigNodeWrapper seedConfigNodeWrapper = + final ConfigNodeWrapper seedConfigNodeWrapper = new ConfigNodeWrapper( true, "", @@ -172,22 +189,23 @@ protected void initEnvironment(int configNodesNum, int dataNodesNum, int retryCo seedConfigNodeWrapper.createLogDir(); seedConfigNodeWrapper.setKillPoints(configNodeKillPoints); seedConfigNodeWrapper.start(); - String seedConfigNode = seedConfigNodeWrapper.getIpAndPortString(); + final String seedConfigNode = seedConfigNodeWrapper.getIpAndPortString(); this.configNodeWrapperList.add(seedConfigNodeWrapper); // Check if the Seed-ConfigNode started successfully - try (SyncConfigNodeIServiceClient ignored = + try (final SyncConfigNodeIServiceClient ignored = (SyncConfigNodeIServiceClient) getLeaderConfigNodeConnection()) { // Do nothing logger.info("The Seed-ConfigNode started successfully!"); - } catch (Exception e) { + } catch (final Exception e) { logger.error("Failed to get connection to the Seed-ConfigNode", e); } - List configNodeEndpoints = new ArrayList<>(); - RequestDelegate configNodesDelegate = new SerialRequestDelegate<>(configNodeEndpoints); + final List configNodeEndpoints = new ArrayList<>(); + final RequestDelegate configNodesDelegate = + new SerialRequestDelegate<>(configNodeEndpoints); for (int i = 1; i < configNodesNum; i++) { - ConfigNodeWrapper configNodeWrapper = + final ConfigNodeWrapper configNodeWrapper = new ConfigNodeWrapper( false, seedConfigNode, @@ -214,16 +232,16 @@ protected void initEnvironment(int configNodesNum, int dataNodesNum, int retryCo } try { configNodesDelegate.requestAll(); - } catch (SQLException e) { + } catch (final SQLException e) { logger.error("Start configNodes failed", e); throw new AssertionError(); } - List dataNodeEndpoints = new ArrayList<>(); - RequestDelegate dataNodesDelegate = + final List dataNodeEndpoints = new ArrayList<>(); + final RequestDelegate dataNodesDelegate = new ParallelRequestDelegate<>(dataNodeEndpoints, NODE_START_TIMEOUT); for (int i = 0; i < dataNodesNum; i++) { - DataNodeWrapper dataNodeWrapper = + final DataNodeWrapper dataNodeWrapper = new DataNodeWrapper( seedConfigNode, testClassName, @@ -250,20 +268,56 @@ protected void initEnvironment(int configNodesNum, int dataNodesNum, int retryCo try { dataNodesDelegate.requestAll(); - } catch (SQLException e) { + } catch (final SQLException e) { logger.error("Start dataNodes failed", e); throw new AssertionError(); } + if (addAINode) { + this.aiNodeWrapperList = new ArrayList<>(); + startAINode(seedConfigNode, testClassName); + } + checkClusterStatusWithoutUnknown(); } + private void startAINode(final String seedConfigNode, final String testClassName) { + final String aiNodeEndPoint; + final AINodeWrapper aiNodeWrapper = + new AINodeWrapper( + seedConfigNode, + testClassName, + testMethodName, + index, + EnvUtils.searchAvailablePorts(), + startTime); + aiNodeWrapperList.add(aiNodeWrapper); + aiNodeEndPoint = aiNodeWrapper.getIpAndPortString(); + aiNodeWrapper.createNodeDir(); + aiNodeWrapper.createLogDir(); + final RequestDelegate aiNodesDelegate = + new ParallelRequestDelegate<>( + Collections.singletonList(aiNodeEndPoint), NODE_START_TIMEOUT); + + aiNodesDelegate.addRequest( + () -> { + aiNodeWrapper.start(); + return null; + }); + + try { + aiNodesDelegate.requestAll(); + } catch (final SQLException e) { + logger.error("Start aiNodes failed", e); + } + } + public String getTestClassName() { - StackTraceElement[] stack = Thread.currentThread().getStackTrace(); - for (StackTraceElement stackTraceElement : stack) { - String className = stackTraceElement.getClassName(); + final StackTraceElement[] stack = Thread.currentThread().getStackTrace(); + for (final StackTraceElement stackTraceElement : stack) { + final String className = stackTraceElement.getClassName(); if (className.endsWith("IT")) { - String result = className.substring(className.lastIndexOf(".") + 1); + final String result = className.substring(className.lastIndexOf(".") + 1); if (!result.startsWith("Abstract")) { return result; } @@ -272,12 +326,16 @@ public String getTestClassName() { return "UNKNOWN-IT"; } - private Map countNodeStatus(Map nodeStatus) { - Map result = new HashMap<>(); + private Map countNodeStatus(final Map nodeStatus) { + final Map result = new HashMap<>(); nodeStatus.values().forEach(status -> result.put(status, result.getOrDefault(status, 0) + 1)); return result; } + public void checkNodeInStatus(int nodeId, NodeStatus expectation) { + checkClusterStatus(nodeStatusMap -> expectation.getStatus().equals(nodeStatusMap.get(nodeId))); + } + public void checkClusterStatusWithoutUnknown() { checkClusterStatus( nodeStatusMap -> nodeStatusMap.values().stream().noneMatch("Unknown"::equals)); @@ -300,13 +358,13 @@ public void checkClusterStatusOneUnknownOtherRunning() { * * @param statusCheck the predicate to test the status of nodes */ - public void checkClusterStatus(Predicate> statusCheck) { + public void checkClusterStatus(final Predicate> statusCheck) { logger.info("Testing cluster environment..."); TShowClusterResp showClusterResp; Exception lastException = null; boolean flag; for (int i = 0; i < retryCount; i++) { - try (SyncConfigNodeIServiceClient client = + try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) getLeaderConfigNodeConnection()) { flag = true; showClusterResp = client.showCluster(); @@ -318,26 +376,27 @@ public void checkClusterStatus(Predicate> statusCheck) { // Check the number of nodes if (showClusterResp.getNodeStatus().size() - != configNodeWrapperList.size() + dataNodeWrapperList.size()) { + != configNodeWrapperList.size() + + dataNodeWrapperList.size() + + aiNodeWrapperList.size()) { flag = false; } // Check the status of nodes if (flag) { - Map nodeStatus = showClusterResp.getNodeStatus(); - flag = statusCheck.test(nodeStatus); + flag = statusCheck.test(showClusterResp.getNodeStatus()); } if (flag) { logger.info("The cluster is now ready for testing!"); return; } - } catch (Exception e) { + } catch (final Exception e) { lastException = e; } try { TimeUnit.SECONDS.sleep(1L); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { lastException = e; Thread.currentThread().interrupt(); } @@ -354,17 +413,19 @@ public void checkClusterStatus(Predicate> statusCheck) { @Override public void cleanClusterEnvironment() { - List allNodeWrappers = - Stream.concat(this.dataNodeWrapperList.stream(), this.configNodeWrapperList.stream()) + final List allNodeWrappers = + Stream.concat( + dataNodeWrapperList.stream(), + Stream.concat(configNodeWrapperList.stream(), aiNodeWrapperList.stream())) .collect(Collectors.toList()); allNodeWrappers.stream() .findAny() .ifPresent( nodeWrapper -> logger.info("You can find logs at {}", nodeWrapper.getLogDirPath())); - for (AbstractNodeWrapper nodeWrapper : allNodeWrappers) { + for (final AbstractNodeWrapper nodeWrapper : allNodeWrappers) { nodeWrapper.stopForcibly(); nodeWrapper.destroyDir(); - String lockPath = EnvUtils.getLockFilePath(nodeWrapper.getPort()); + final String lockPath = EnvUtils.getLockFilePath(nodeWrapper.getPort()); if (!new File(lockPath).delete()) { logger.error("Delete lock file {} failed", lockPath); } @@ -377,89 +438,213 @@ public void cleanClusterEnvironment() { } @Override - public Connection getConnection(String username, String password) throws SQLException { + public Connection getConnection( + final String username, final String password, final String sqlDialect) throws SQLException { + return new ClusterTestConnection( + getWriteConnection(null, username, password, sqlDialect), + getReadConnections(null, username, password, sqlDialect)); + } + + @Override + public Connection getConnection( + final DataNodeWrapper dataNodeWrapper, + final String username, + final String password, + final String sqlDialect) + throws SQLException { return new ClusterTestConnection( - getWriteConnection(null, username, password), getReadConnections(null, username, password)); + getWriteConnectionWithSpecifiedDataNode( + dataNodeWrapper, null, username, password, sqlDialect), + getReadConnections(null, dataNodeWrapper, username, password, sqlDialect)); } @Override public Connection getWriteOnlyConnectionWithSpecifiedDataNode( - DataNodeWrapper dataNode, String username, String password) throws SQLException { + final DataNodeWrapper dataNode, + final String username, + final String password, + String sqlDialect) + throws SQLException { return new ClusterTestConnection( - getWriteConnectionWithSpecifiedDataNode(dataNode, null, username, password), + getWriteConnectionWithSpecifiedDataNode( + dataNode, + null, + username, + password, + TABLE_SQL_DIALECT.equals(sqlDialect) ? TABLE_SQL_DIALECT : TREE_SQL_DIALECT), Collections.emptyList()); } @Override public Connection getConnectionWithSpecifiedDataNode( - DataNodeWrapper dataNode, String username, String password) throws SQLException { + final DataNodeWrapper dataNode, final String username, final String password) + throws SQLException { return new ClusterTestConnection( - getWriteConnectionWithSpecifiedDataNode(dataNode, null, username, password), - getReadConnections(null, username, password)); + getWriteConnectionWithSpecifiedDataNode( + dataNode, null, username, password, TREE_SQL_DIALECT), + getReadConnections(null, username, password, TREE_SQL_DIALECT)); } @Override - public Connection getConnection(Constant.Version version, String username, String password) + public Connection getConnection( + final Constant.Version version, + final String username, + final String password, + final String sqlDialect) throws SQLException { - if (System.getProperty("ReadAndVerifyWithMultiNode", "true").equalsIgnoreCase("true")) { - return new ClusterTestConnection( - getWriteConnection(version, username, password), - getReadConnections(version, username, password)); - } else { - return getWriteConnection(version, username, password).getUnderlyingConnecton(); - } + return System.getProperty("ReadAndVerifyWithMultiNode", "true").equalsIgnoreCase("true") + ? new ClusterTestConnection( + getWriteConnection(version, username, password, sqlDialect), + getReadConnections(version, username, password, sqlDialect)) + : getWriteConnection(version, username, password, sqlDialect).getUnderlyingConnecton(); } @Override public ISession getSessionConnection() throws IoTDBConnectionException { - DataNodeWrapper dataNode = + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + final Session session = + new Session.Builder().host(dataNode.getIp()).port(dataNode.getPort()).build(); + session.open(); + return session; + } + + @Override + public ISession getSessionConnection(ZoneId zoneId) throws IoTDBConnectionException { + final DataNodeWrapper dataNode = this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); - Session session = new Session(dataNode.getIp(), dataNode.getPort()); + final Session session = + new Session.Builder() + .host(dataNode.getIp()) + .port(dataNode.getPort()) + .zoneId(zoneId) + .build(); session.open(); return session; } @Override - public ISession getSessionConnection(String userName, String password) + public ISession getSessionConnection(final String userName, final String password) throws IoTDBConnectionException { - DataNodeWrapper dataNode = + final DataNodeWrapper dataNode = this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); - Session session = new Session(dataNode.getIp(), dataNode.getPort(), userName, password); + final Session session = + new Session.Builder() + .host(dataNode.getIp()) + .port(dataNode.getPort()) + .username(userName) + .password(password) + .build(); session.open(); return session; } @Override - public ISession getSessionConnection(List nodeUrls) throws IoTDBConnectionException { - Session session = - new Session( - nodeUrls, - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - SessionConfig.DEFAULT_FETCH_SIZE, - null, - SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY, - SessionConfig.DEFAULT_MAX_FRAME_SIZE, - SessionConfig.DEFAULT_REDIRECTION_MODE, - SessionConfig.DEFAULT_VERSION); + public ISession getSessionConnection(final List nodeUrls) + throws IoTDBConnectionException { + final Session session = + new Session.Builder() + .nodeUrls(nodeUrls) + .username(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .zoneId(null) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .version(SessionConfig.DEFAULT_VERSION) + .build(); session.open(); return session; } @Override - public ISessionPool getSessionPool(int maxSize) { + public ITableSession getTableSessionConnection() throws IoTDBConnectionException { + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) + .build(); + } + + @Override + public ITableSession getTableSessionConnection(String userName, String password) + throws IoTDBConnectionException { + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) + .username(userName) + .password(password) + .build(); + } + + @Override + public ITableSession getTableSessionConnectionWithDB(final String database) + throws IoTDBConnectionException { + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) + .database(database) + .build(); + } + + public ITableSession getTableSessionConnection(List nodeUrls) + throws IoTDBConnectionException { + return new TableSessionBuilder() + .nodeUrls(nodeUrls) + .username(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .zoneId(null) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .build(); + } + + @Override + public ISessionPool getSessionPool(final int maxSize) { + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return new SessionPool.Builder() + .host(dataNode.getIp()) + .port(dataNode.getPort()) + .user(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .maxSize(maxSize) + .build(); + } + + @Override + public ITableSessionPool getTableSessionPool(final int maxSize) { + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) + .user(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .maxSize(maxSize) + .build(); + } + + @Override + public ITableSessionPool getTableSessionPool(final int maxSize, final String database) { DataNodeWrapper dataNode = this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); - return new SessionPool( - dataNode.getIp(), - dataNode.getPort(), - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - maxSize); + return new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) + .user(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .database(database) + .maxSize(maxSize) + .build(); } protected NodeConnection getWriteConnection( - Constant.Version version, String username, String password) throws SQLException { + Constant.Version version, String username, String password, String sqlDialect) + throws SQLException { DataNodeWrapper dataNode; if (System.getProperty("RandomSelectWriteNode", "true").equalsIgnoreCase("true")) { @@ -470,20 +655,23 @@ protected NodeConnection getWriteConnection( } return getWriteConnectionFromDataNodeList( - this.dataNodeWrapperList, version, username, password); + this.dataNodeWrapperList, version, username, password, sqlDialect); } protected NodeConnection getWriteConnectionWithSpecifiedDataNode( - DataNodeWrapper dataNode, Constant.Version version, String username, String password) + final DataNodeWrapper dataNode, + final Constant.Version version, + final String username, + final String password, + final String sqlDialect) throws SQLException { - String endpoint = dataNode.getIp() + ":" + dataNode.getPort(); - Connection writeConnection = + final String endpoint = dataNode.getIp() + ":" + dataNode.getPort(); + final Connection writeConnection = DriverManager.getConnection( Config.IOTDB_URL_PREFIX + endpoint + getParam(version, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE), - System.getProperty("User", username), - System.getProperty("Password", password)); + BaseEnv.constructProperties(username, password, sqlDialect)); return new NodeConnection( endpoint, NodeConnection.NodeRole.DATA_NODE, @@ -492,49 +680,83 @@ protected NodeConnection getWriteConnectionWithSpecifiedDataNode( } protected NodeConnection getWriteConnectionFromDataNodeList( - List dataNodeList, - Constant.Version version, - String username, - String password) + final List dataNodeList, + final Constant.Version version, + final String username, + final String password, + final String sqlDialect) throws SQLException { - List dataNodeWrapperListCopy = new ArrayList<>(dataNodeList); + final List dataNodeWrapperListCopy = new ArrayList<>(dataNodeList); Collections.shuffle(dataNodeWrapperListCopy); SQLException lastException = null; - for (DataNodeWrapper dataNode : dataNodeWrapperListCopy) { + for (final DataNodeWrapper dataNode : dataNodeWrapperListCopy) { try { - return getWriteConnectionWithSpecifiedDataNode(dataNode, version, username, password); - } catch (SQLException e) { + return getWriteConnectionWithSpecifiedDataNode( + dataNode, version, username, password, sqlDialect); + } catch (final SQLException e) { lastException = e; } } - logger.error("Failed to get connection from any DataNode, last exception is ", lastException); + if (!(lastException.getCause() instanceof TTransportException)) { + logger.error("Failed to get connection from any DataNode, last exception is ", lastException); + } throw lastException; } protected List getReadConnections( - Constant.Version version, String username, String password) throws SQLException { - List endpoints = new ArrayList<>(); - ParallelRequestDelegate readConnRequestDelegate = + final Constant.Version version, + final String username, + final String password, + final String sqlDialect) + throws SQLException { + final List endpoints = new ArrayList<>(); + final ParallelRequestDelegate readConnRequestDelegate = new ParallelRequestDelegate<>(endpoints, NODE_START_TIMEOUT); - for (DataNodeWrapper dataNodeWrapper : this.dataNodeWrapperList) { - final String endpoint = dataNodeWrapper.getIpAndPortString(); - endpoints.add(endpoint); - readConnRequestDelegate.addRequest( - () -> { - Connection readConnection = + + dataNodeWrapperList.stream() + .map(AbstractNodeWrapper::getIpAndPortString) + .forEach( + endpoint -> { + endpoints.add(endpoint); + readConnRequestDelegate.addRequest( + () -> + new NodeConnection( + endpoint, + NodeConnection.NodeRole.DATA_NODE, + NodeConnection.ConnectionRole.READ, + DriverManager.getConnection( + Config.IOTDB_URL_PREFIX + + endpoint + + getParam(version, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE), + BaseEnv.constructProperties(username, password, sqlDialect)))); + }); + return readConnRequestDelegate.requestAll(); + } + + protected List getReadConnections( + final Constant.Version version, + final DataNodeWrapper dataNode, + final String username, + final String password, + final String sqlDialect) + throws SQLException { + final List endpoints = new ArrayList<>(); + final ParallelRequestDelegate readConnRequestDelegate = + new ParallelRequestDelegate<>(endpoints, NODE_START_TIMEOUT); + + endpoints.add(dataNode.getIpAndPortString()); + readConnRequestDelegate.addRequest( + () -> + new NodeConnection( + dataNode.getIpAndPortString(), + NodeConnection.NodeRole.DATA_NODE, + NodeConnection.ConnectionRole.READ, DriverManager.getConnection( Config.IOTDB_URL_PREFIX - + endpoint + + dataNode.getIpAndPortString() + getParam(version, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE), - System.getProperty("User", username), - System.getProperty("Password", password)); - return new NodeConnection( - endpoint, - NodeConnection.NodeRole.DATA_NODE, - NodeConnection.ConnectionRole.READ, - readConnection); - }); - } + BaseEnv.constructProperties(username, password, sqlDialect)))); + return readConnRequestDelegate.requestAll(); } @@ -546,19 +768,19 @@ protected List getReadConnections( // AssertionError. protected void testJDBCConnection() { logger.info("Testing JDBC connection..."); - List endpoints = + final List endpoints = dataNodeWrapperList.stream() .map(DataNodeWrapper::getIpAndPortString) .collect(Collectors.toList()); - RequestDelegate testDelegate = + final RequestDelegate testDelegate = new ParallelRequestDelegate<>(endpoints, NODE_START_TIMEOUT); - for (DataNodeWrapper dataNode : dataNodeWrapperList) { + for (final DataNodeWrapper dataNode : dataNodeWrapperList) { final String dataNodeEndpoint = dataNode.getIpAndPortString(); testDelegate.addRequest( () -> { Exception lastException = null; for (int i = 0; i < retryCount; i++) { - try (IoTDBConnection ignored = + try (final IoTDBConnection ignored = (IoTDBConnection) DriverManager.getConnection( Config.IOTDB_URL_PREFIX @@ -568,7 +790,7 @@ protected void testJDBCConnection() { System.getProperty("Password", "root"))) { logger.info("Successfully connecting to DataNode: {}.", dataNodeEndpoint); return null; - } catch (Exception e) { + } catch (final Exception e) { lastException = e; TimeUnit.SECONDS.sleep(1L); } @@ -581,15 +803,16 @@ protected void testJDBCConnection() { } try { testDelegate.requestAll(); - } catch (Exception e) { + } catch (final Exception e) { logger.error("exception in test Cluster with RPC, message: {}", e.getMessage(), e); throw new AssertionError( String.format("After %d times retry, the cluster can't work!", retryCount)); } } - private String getParam(Constant.Version version, int timeout, String timeZone) { - StringBuilder sb = new StringBuilder("?"); + private String getParam( + final Constant.Version version, final int timeout, final String timeZone) { + final StringBuilder sb = new StringBuilder("?"); sb.append(Config.NETWORK_TIMEOUT).append("=").append(timeout); if (version != null) { sb.append("&").append(VERSION).append("=").append(version); @@ -605,23 +828,20 @@ public String getTestMethodName() { } @Override - public void setTestMethodName(String testMethodName) { + public void setTestMethodName(final String testMethodName) { this.testMethodName = testMethodName; } @Override public void dumpTestJVMSnapshot() { - for (ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { - configNodeWrapper.executeJstack(testMethodName); - } - for (DataNodeWrapper dataNodeWrapper : dataNodeWrapperList) { - dataNodeWrapper.executeJstack(testMethodName); - } + configNodeWrapperList.forEach( + configNodeWrapper -> configNodeWrapper.executeJstack(testMethodName)); + dataNodeWrapperList.forEach(dataNodeWrapper -> dataNodeWrapper.executeJstack(testMethodName)); } @Override public List getNodeWrapperList() { - List result = new ArrayList<>(configNodeWrapperList); + final List result = new ArrayList<>(configNodeWrapperList); result.addAll(dataNodeWrapperList); return result; } @@ -650,13 +870,13 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() Exception lastException = null; ConfigNodeWrapper lastErrorNode = null; for (int i = 0; i < retryCount; i++) { - for (ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { + for (final ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { try { lastErrorNode = configNodeWrapper; - SyncConfigNodeIServiceClient client = + final SyncConfigNodeIServiceClient client = clientManager.borrowClient( new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort())); - TShowClusterResp resp = client.showCluster(); + final TShowClusterResp resp = client.showCluster(); if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // Only the ConfigNodeClient who connects to the ConfigNode-leader @@ -671,7 +891,7 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() + " message: " + resp.getStatus().getMessage()); } - } catch (Exception e) { + } catch (final Exception e) { lastException = e; } @@ -692,12 +912,12 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() @Override public IConfigNodeRPCService.Iface getConfigNodeConnection(int index) throws Exception { Exception lastException = null; - ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(index); + final ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(index); for (int i = 0; i < 30; i++) { try { return clientManager.borrowClient( new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort())); - } catch (Exception e) { + } catch (final Exception e) { lastException = e; } // Sleep 1s before next retry @@ -713,9 +933,9 @@ public int getFirstLeaderSchemaRegionDataNodeIndex() throws IOException, Interru ConfigNodeWrapper lastErrorNode = null; for (int retry = 0; retry < 30; retry++) { for (int configNodeId = 0; configNodeId < configNodeWrapperList.size(); configNodeId++) { - ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(configNodeId); + final ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(configNodeId); lastErrorNode = configNodeWrapper; - try (SyncConfigNodeIServiceClient client = + try (final SyncConfigNodeIServiceClient client = clientManager.borrowClient( new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort()))) { TShowRegionResp resp = @@ -728,12 +948,12 @@ public int getFirstLeaderSchemaRegionDataNodeIndex() throws IOException, Interru int port; if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - for (TRegionInfo tRegionInfo : resp.getRegionInfoList()) { + for (final TRegionInfo tRegionInfo : resp.getRegionInfoList()) { if (tRegionInfo.getRoleType().equals("Leader")) { ip = tRegionInfo.getClientRpcIp(); port = tRegionInfo.getClientRpcPort(); for (int dataNodeId = 0; dataNodeId < dataNodeWrapperList.size(); ++dataNodeId) { - DataNodeWrapper dataNodeWrapper = dataNodeWrapperList.get(dataNodeId); + final DataNodeWrapper dataNodeWrapper = dataNodeWrapperList.get(dataNodeId); if (dataNodeWrapper.getIp().equals(ip) && dataNodeWrapper.getPort() == port) { return dataNodeId; } @@ -749,7 +969,7 @@ public int getFirstLeaderSchemaRegionDataNodeIndex() throws IOException, Interru + " message: " + resp.getStatus().getMessage()); } - } catch (Exception e) { + } catch (final Exception e) { lastException = e; } @@ -773,12 +993,12 @@ public int getLeaderConfigNodeIndex() throws IOException, InterruptedException { ConfigNodeWrapper lastErrorNode = null; for (int retry = 0; retry < retryCount; retry++) { for (int configNodeId = 0; configNodeId < configNodeWrapperList.size(); configNodeId++) { - ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(configNodeId); + final ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(configNodeId); lastErrorNode = configNodeWrapper; - try (SyncConfigNodeIServiceClient client = + try (final SyncConfigNodeIServiceClient client = clientManager.borrowClient( new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort()))) { - TShowClusterResp resp = client.showCluster(); + final TShowClusterResp resp = client.showCluster(); // Only the ConfigNodeClient who connects to the ConfigNode-leader // will respond the SUCCESS_STATUS if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -790,7 +1010,7 @@ public int getLeaderConfigNodeIndex() throws IOException, InterruptedException { + " message: " + resp.getStatus().getMessage()); } - } catch (Exception e) { + } catch (final Exception e) { lastException = e; } @@ -810,15 +1030,13 @@ public int getLeaderConfigNodeIndex() throws IOException, InterruptedException { } @Override - public void startConfigNode(int index) { + public void startConfigNode(final int index) { configNodeWrapperList.get(index).start(); } @Override public void startAllConfigNodes() { - for (ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { - configNodeWrapper.start(); - } + configNodeWrapperList.forEach(AbstractNodeWrapper::start); } @Override @@ -828,24 +1046,27 @@ public void shutdownConfigNode(int index) { @Override public void shutdownAllConfigNodes() { - for (ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { - configNodeWrapper.stop(); - } + configNodeWrapperList.forEach(AbstractNodeWrapper::stop); + } + + @Override + public void shutdownForciblyAllConfigNodes() { + configNodeWrapperList.forEach(AbstractNodeWrapper::stopForcibly); } @Override - public ConfigNodeWrapper getConfigNodeWrapper(int index) { + public ConfigNodeWrapper getConfigNodeWrapper(final int index) { return configNodeWrapperList.get(index); } @Override - public DataNodeWrapper getDataNodeWrapper(int index) { + public DataNodeWrapper getDataNodeWrapper(final int index) { return dataNodeWrapperList.get(index); } @Override public ConfigNodeWrapper generateRandomConfigNodeWrapper() { - ConfigNodeWrapper newConfigNodeWrapper = + final ConfigNodeWrapper newConfigNodeWrapper = new ConfigNodeWrapper( false, configNodeWrapperList.get(0).getIpAndPortString(), @@ -867,7 +1088,7 @@ public ConfigNodeWrapper generateRandomConfigNodeWrapper() { @Override public DataNodeWrapper generateRandomDataNodeWrapper() { - DataNodeWrapper newDataNodeWrapper = + final DataNodeWrapper newDataNodeWrapper = new DataNodeWrapper( configNodeWrapperList.get(0).getIpAndPortString(), getTestClassName(), @@ -887,19 +1108,20 @@ public DataNodeWrapper generateRandomDataNodeWrapper() { } @Override - public void registerNewDataNode(boolean isNeedVerify) { + public void registerNewDataNode(final boolean isNeedVerify) { registerNewDataNode(generateRandomDataNodeWrapper(), isNeedVerify); } @Override - public void registerNewConfigNode(boolean isNeedVerify) { + public void registerNewConfigNode(final boolean isNeedVerify) { registerNewConfigNode(generateRandomConfigNodeWrapper(), isNeedVerify); } @Override - public void registerNewConfigNode(ConfigNodeWrapper newConfigNodeWrapper, boolean isNeedVerify) { + public void registerNewConfigNode( + final ConfigNodeWrapper newConfigNodeWrapper, final boolean isNeedVerify) { // Start new ConfigNode - RequestDelegate configNodeDelegate = + final RequestDelegate configNodeDelegate = new ParallelRequestDelegate<>( Collections.singletonList(newConfigNodeWrapper.getIpAndPortString()), NODE_START_TIMEOUT); @@ -911,7 +1133,7 @@ public void registerNewConfigNode(ConfigNodeWrapper newConfigNodeWrapper, boolea try { configNodeDelegate.requestAll(); - } catch (SQLException e) { + } catch (final SQLException e) { logger.error("Start configNode failed", e); throw new AssertionError(); } @@ -923,11 +1145,12 @@ public void registerNewConfigNode(ConfigNodeWrapper newConfigNodeWrapper, boolea } @Override - public void registerNewDataNode(DataNodeWrapper newDataNodeWrapper, boolean isNeedVerify) { + public void registerNewDataNode( + final DataNodeWrapper newDataNodeWrapper, final boolean isNeedVerify) { // Start new DataNode - List dataNodeEndpoints = + final List dataNodeEndpoints = Collections.singletonList(newDataNodeWrapper.getIpAndPortString()); - RequestDelegate dataNodesDelegate = + final RequestDelegate dataNodesDelegate = new ParallelRequestDelegate<>(dataNodeEndpoints, NODE_START_TIMEOUT); dataNodesDelegate.addRequest( () -> { @@ -936,7 +1159,7 @@ public void registerNewDataNode(DataNodeWrapper newDataNodeWrapper, boolean isNe }); try { dataNodesDelegate.requestAll(); - } catch (SQLException e) { + } catch (final SQLException e) { logger.error("Start dataNodes failed", e); throw new AssertionError(); } @@ -948,58 +1171,68 @@ public void registerNewDataNode(DataNodeWrapper newDataNodeWrapper, boolean isNe } @Override - public void startDataNode(int index) { + public void startDataNode(final int index) { dataNodeWrapperList.get(index).start(); } @Override public void startAllDataNodes() { - for (DataNodeWrapper dataNodeWrapper : dataNodeWrapperList) { - dataNodeWrapper.start(); - } + dataNodeWrapperList.forEach(AbstractNodeWrapper::start); } @Override - public void shutdownDataNode(int index) { + public void shutdownDataNode(final int index) { dataNodeWrapperList.get(index).stop(); } @Override public void shutdownAllDataNodes() { - for (DataNodeWrapper dataNodeWrapper : dataNodeWrapperList) { - dataNodeWrapper.stop(); - } + dataNodeWrapperList.forEach(AbstractNodeWrapper::stop); + } + + @Override + public void shutdownForciblyAllDataNodes() { + dataNodeWrapperList.forEach(AbstractNodeWrapper::stopForcibly); } @Override - public void ensureNodeStatus(List nodes, List targetStatus) + public void ensureNodeStatus( + final List nodes, final List targetStatus) throws IllegalStateException { Throwable lastException = null; for (int i = 0; i < retryCount; i++) { - try (SyncConfigNodeIServiceClient client = + try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - List errorMessages = new ArrayList<>(nodes.size()); - Map nodeIds = new HashMap<>(nodes.size()); - TShowClusterResp showClusterResp = client.showCluster(); - for (TConfigNodeLocation node : showClusterResp.getConfigNodeList()) { - nodeIds.put( - node.getInternalEndPoint().getIp() + ":" + node.getInternalEndPoint().getPort(), - node.getConfigNodeId()); - } - for (TDataNodeLocation node : showClusterResp.getDataNodeList()) { - nodeIds.put( - node.getClientRpcEndPoint().getIp() + ":" + node.getClientRpcEndPoint().getPort(), - node.getDataNodeId()); - } + final List errorMessages = new ArrayList<>(nodes.size()); + final Map nodeIds = new HashMap<>(nodes.size()); + final TShowClusterResp showClusterResp = client.showCluster(); + showClusterResp + .getConfigNodeList() + .forEach( + node -> + nodeIds.put( + node.getInternalEndPoint().getIp() + + ":" + + node.getInternalEndPoint().getPort(), + node.getConfigNodeId())); + showClusterResp + .getDataNodeList() + .forEach( + node -> + nodeIds.put( + node.getClientRpcEndPoint().getIp() + + ":" + + node.getClientRpcEndPoint().getPort(), + node.getDataNodeId())); for (int j = 0; j < nodes.size(); j++) { - String endpoint = nodes.get(j).getIpAndPortString(); + final String endpoint = nodes.get(j).getIpAndPortString(); if (!nodeIds.containsKey(endpoint)) { // Node not exist // Notice: Never modify this line, since the NodeLocation might be modified in IT errorMessages.add("The node " + nodes.get(j).getIpAndPortString() + " is not found!"); continue; } - String status = showClusterResp.getNodeStatus().get(nodeIds.get(endpoint)); + final String status = showClusterResp.getNodeStatus().get(nodeIds.get(endpoint)); if (!targetStatus.get(j).getStatus().equals(status)) { // Error status errorMessages.add( @@ -1013,12 +1246,12 @@ public void ensureNodeStatus(List nodes, List targe } else { lastException = new IllegalStateException(String.join(". ", errorMessages)); } - } catch (TException | ClientManagerException | IOException | InterruptedException e) { + } catch (final TException | ClientManagerException | IOException | InterruptedException e) { lastException = e; } try { TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { throw new RuntimeException(e); } } @@ -1027,8 +1260,9 @@ public void ensureNodeStatus(List nodes, List targe @Override public int getMqttPort() { - int randomIndex = new Random(System.currentTimeMillis()).nextInt(dataNodeWrapperList.size()); - return dataNodeWrapperList.get(randomIndex).getMqttPort(); + return dataNodeWrapperList + .get(new Random(System.currentTimeMillis()).nextInt(dataNodeWrapperList.size())) + .getMqttPort(); } @Override @@ -1057,11 +1291,11 @@ public String getLibPath() { } @Override - public Optional dataNodeIdToWrapper(int nodeId) { - try (SyncConfigNodeIServiceClient leaderClient = + public Optional dataNodeIdToWrapper(final int nodeId) { + try (final SyncConfigNodeIServiceClient leaderClient = (SyncConfigNodeIServiceClient) getLeaderConfigNodeConnection()) { - TShowDataNodesResp resp = leaderClient.showDataNodes(); - for (TDataNodeInfo dataNodeInfo : resp.getDataNodesInfoList()) { + final TShowDataNodesResp resp = leaderClient.showDataNodes(); + for (final TDataNodeInfo dataNodeInfo : resp.getDataNodesInfoList()) { if (dataNodeInfo.getDataNodeId() == nodeId) { return dataNodeWrapperList.stream() .filter(dataNodeWrapper -> dataNodeWrapper.getPort() == dataNodeInfo.getRpcPort()) @@ -1069,18 +1303,18 @@ public Optional dataNodeIdToWrapper(int nodeId) { } } return Optional.empty(); - } catch (Exception e) { + } catch (final Exception e) { return Optional.empty(); } } @Override - public void registerConfigNodeKillPoints(List killPoints) { + public void registerConfigNodeKillPoints(final List killPoints) { this.configNodeKillPoints = killPoints; } @Override - public void registerDataNodeKillPoints(List killPoints) { + public void registerDataNodeKillPoints(final List killPoints) { this.dataNodeKillPoints = killPoints; } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MultiClusterEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MultiClusterEnv.java index 1eb05cb315f41..d462b5a668b3b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MultiClusterEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MultiClusterEnv.java @@ -25,7 +25,7 @@ public class MultiClusterEnv extends AbstractEnv { - public MultiClusterEnv(long startTime, int index, String currentMethodName) { + public MultiClusterEnv(final long startTime, final int index, final String currentMethodName) { super(startTime); this.index = index; this.testMethodName = currentMethodName; @@ -33,18 +33,18 @@ public MultiClusterEnv(long startTime, int index, String currentMethodName) { @Override public void initClusterEnvironment() { - Pair nodeNum = EnvUtils.getNodeNum(index); + final Pair nodeNum = EnvUtils.getNodeNum(index); super.initEnvironment(nodeNum.getLeft(), nodeNum.getRight()); } @Override - public void initClusterEnvironment(int configNodesNum, int dataNodesNum) { + public void initClusterEnvironment(final int configNodesNum, final int dataNodesNum) { super.initEnvironment(configNodesNum, dataNodesNum); } @Override public void initClusterEnvironment( - int configNodesNum, int dataNodesNum, int testWorkingRetryCount) { + final int configNodesNum, final int dataNodesNum, final int testWorkingRetryCount) { super.initEnvironment(configNodesNum, dataNodesNum, testWorkingRetryCount); } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java new file mode 100644 index 0000000000000..8da2437aed687 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.it.env.cluster.node; + +import org.apache.iotdb.it.env.cluster.config.MppJVMConfig; +import org.apache.iotdb.it.framework.IoTDBTestLogger; + +import org.slf4j.Logger; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.it.env.cluster.ClusterConstant.AI_NODE_NAME; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PYTHON_PATH; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR; +import static org.apache.iotdb.it.env.cluster.EnvUtils.getTimeForLogDirectory; + +public class AINodeWrapper extends AbstractNodeWrapper { + + private static final Logger logger = IoTDBTestLogger.logger; + private final long startTime; + private final String seedConfigNode; + + private static final String SCRIPT_FILE = "start-ainode.sh"; + + private static final String SHELL_COMMAND = "bash"; + + private static final String PROPERTIES_FILE = "iotdb-ainode.properties"; + public static final String CONFIG_PATH = "conf"; + public static final String SCRIPT_PATH = "sbin"; + + private void replaceAttribute(String[] keys, String[] values, String filePath) { + try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath, true))) { + for (int i = 0; i < keys.length; i++) { + String line = keys[i] + "=" + values[i]; + writer.newLine(); + writer.write(line); + } + } catch (IOException e) { + logger.error( + "Failed to set attribute for AINode in file: {} because {}", filePath, e.getMessage()); + } + } + + public AINodeWrapper( + String seedConfigNode, + String testClassName, + String testMethodName, + int clusterIndex, + int[] port, + long startTime) { + super(testClassName, testMethodName, port, clusterIndex, false, startTime); + this.seedConfigNode = seedConfigNode; + this.startTime = startTime; + } + + @Override + public String getLogDirPath() { + return System.getProperty(USER_DIR) + + File.separator + + TARGET + + File.separator + + "ainode-logs" + + File.separator + + getTestLogDirName() + + File.separator + + getTimeForLogDirectory(startTime); + } + + @Override + String getNodeType() { + return "ainode"; + } + + @Override + public void start() { + try { + File stdoutFile = new File(getLogPath()); + String filePrefix = + System.getProperty(USER_DIR) + + File.separator + + TARGET + + File.separator + + AI_NODE_NAME + + getPort(); + String propertiesFile = + filePrefix + File.separator + CONFIG_PATH + File.separator + PROPERTIES_FILE; + + // set attribute + replaceAttribute( + new String[] {"ain_seed_config_node", "ain_inference_rpc_port"}, + new String[] {this.seedConfigNode, Integer.toString(getPort())}, + propertiesFile); + + // start AINode + List startCommand = new ArrayList<>(); + startCommand.add(SHELL_COMMAND); + startCommand.add(filePrefix + File.separator + SCRIPT_PATH + File.separator + SCRIPT_FILE); + startCommand.add("-i"); + startCommand.add(filePrefix + File.separator + PYTHON_PATH); + startCommand.add("-r"); + + ProcessBuilder processBuilder = + new ProcessBuilder(startCommand) + .redirectOutput(ProcessBuilder.Redirect.appendTo(stdoutFile)) + .redirectError(ProcessBuilder.Redirect.appendTo(stdoutFile)); + this.instance = processBuilder.start(); + logger.info("In test {} {} started.", getTestLogDirName(), getId()); + } catch (Exception e) { + throw new AssertionError("Start AI Node failed. " + e + Paths.get("")); + } + } + + @Override + public int getMetricPort() { + // no metric currently + return -1; + } + + @Override + public String getId() { + return AI_NODE_NAME + getPort(); + } + + /* Abstract methods, which must be implemented in ConfigNode and DataNode. */ + public void reloadMutableFields() {} + ; + + public void renameFile() {} + ; + + public String getSystemConfigPath() { + return ""; + } + ; + + /** Return the node config file path specified through system variable */ + public String getDefaultNodeConfigPath() { + return ""; + } + ; + + /** Return the common config file path specified through system variable */ + public String getDefaultCommonConfigPath() { + return ""; + } + ; + + public void addStartCmdParams(List params) {} + ; + + public String getSystemPropertiesPath() { + return ""; + } + ; + + public MppJVMConfig initVMConfig() { + return null; + } + ; +} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 942b6a9e01242..3b6d840698190 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -20,6 +20,7 @@ package org.apache.iotdb.it.env.cluster.node; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; import org.apache.iotdb.it.env.cluster.config.MppCommonConfig; @@ -79,6 +80,7 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.HIGH_PERFORMANCE_MODE_SCHEMA_REGION_REPLICA_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.HYPHEN; import static org.apache.iotdb.it.env.cluster.ClusterConstant.INFLUXDB_RPC_PORT; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOT_CONSENSUS_V2_MODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.JAVA_CMD; import static org.apache.iotdb.it.env.cluster.ClusterConstant.LIGHT_WEIGHT_STANDALONE_MODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.LIGHT_WEIGHT_STANDALONE_MODE_CONFIG_NODE_CONSENSUS; @@ -88,6 +90,18 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.LIGHT_WEIGHT_STANDALONE_MODE_SCHEMA_REGION_REPLICA_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_HOST; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_PORT; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_CONFIG_NODE_CONSENSUS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_DATA_REGION_CONSENSUS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_DATA_REGION_REPLICA_NUM; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_SCHEMA_REGION_CONSENSUS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_BATCH_MODE_SCHEMA_REGION_REPLICA_NUM; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_CONFIG_NODE_CONSENSUS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_DATA_REGION_CONSENSUS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_DATA_REGION_REPLICA_NUM; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_SCHEMA_REGION_CONSENSUS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_CONSENSUS_STREAM_MODE_SCHEMA_REGION_REPLICA_NUM; import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_LIB_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.REST_SERVICE_PORT; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCALABLE_SINGLE_NODE_MODE; @@ -125,11 +139,11 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { protected final MppJVMConfig jvmConfig; protected final int clusterIndex; protected final boolean isMultiCluster; - private Process instance; + protected Process instance; private final String nodeAddress; private int nodePort; - private int metricPort; - private long startTime; + private final int metricPort; + private final long startTime; private List killPoints = new ArrayList<>(); /** @@ -401,6 +415,52 @@ private void reloadClusterConfigurations() { DATA_REPLICATION_FACTOR, System.getProperty(STRONG_CONSISTENCY_CLUSTER_MODE_DATA_REGION_REPLICA_NUM)); break; + case PIPE_CONSENSUS_BATCH_MODE: + clusterConfigProperties.setProperty( + CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS, + fromConsensusAbbrToFullName( + System.getProperty(PIPE_CONSENSUS_BATCH_MODE_CONFIG_NODE_CONSENSUS))); + clusterConfigProperties.setProperty( + SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS, + fromConsensusAbbrToFullName( + System.getProperty(PIPE_CONSENSUS_BATCH_MODE_SCHEMA_REGION_CONSENSUS))); + clusterConfigProperties.setProperty( + DATA_REGION_CONSENSUS_PROTOCOL_CLASS, + fromConsensusAbbrToFullName( + System.getProperty(PIPE_CONSENSUS_BATCH_MODE_DATA_REGION_CONSENSUS))); + clusterConfigProperties.setProperty( + SCHEMA_REPLICATION_FACTOR, + System.getProperty(PIPE_CONSENSUS_BATCH_MODE_SCHEMA_REGION_REPLICA_NUM)); + clusterConfigProperties.setProperty( + DATA_REPLICATION_FACTOR, + System.getProperty(PIPE_CONSENSUS_BATCH_MODE_DATA_REGION_REPLICA_NUM)); + // set mode + clusterConfigProperties.setProperty( + IOT_CONSENSUS_V2_MODE, ConsensusFactory.IOT_CONSENSUS_V2_BATCH_MODE); + break; + case PIPE_CONSENSUS_STREAM_MODE: + clusterConfigProperties.setProperty( + CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS, + fromConsensusAbbrToFullName( + System.getProperty(PIPE_CONSENSUS_STREAM_MODE_CONFIG_NODE_CONSENSUS))); + clusterConfigProperties.setProperty( + SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS, + fromConsensusAbbrToFullName( + System.getProperty(PIPE_CONSENSUS_STREAM_MODE_SCHEMA_REGION_CONSENSUS))); + clusterConfigProperties.setProperty( + DATA_REGION_CONSENSUS_PROTOCOL_CLASS, + fromConsensusAbbrToFullName( + System.getProperty(PIPE_CONSENSUS_STREAM_MODE_DATA_REGION_CONSENSUS))); + clusterConfigProperties.setProperty( + SCHEMA_REPLICATION_FACTOR, + System.getProperty(PIPE_CONSENSUS_STREAM_MODE_SCHEMA_REGION_REPLICA_NUM)); + clusterConfigProperties.setProperty( + DATA_REPLICATION_FACTOR, + System.getProperty(PIPE_CONSENSUS_STREAM_MODE_DATA_REGION_REPLICA_NUM)); + // set mode + clusterConfigProperties.setProperty( + IOT_CONSENSUS_V2_MODE, ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + break; default: // Print nothing to avoid polluting test outputs } @@ -523,7 +583,7 @@ public final int getPort() { } @Override - public final int getMetricPort() { + public int getMetricPort() { return this.metricPort; } @@ -540,7 +600,7 @@ protected String workDirFilePath(String dirName, String fileName) { return getNodePath() + File.separator + dirName + File.separator + fileName; } - private String getLogPath() { + protected String getLogPath() { return getLogDirPath() + File.separator + getId() + ".log"; } @@ -570,6 +630,18 @@ public String getNodePath() { return System.getProperty(USER_DIR) + File.separator + TARGET + File.separator + getId(); } + public String getDataPath() { + return getNodePath() + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + getNodeType() + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME; + } + + abstract String getNodeType(); + public void dumpJVMSnapshot(String testCaseName) { JMXServiceURL url; try { @@ -645,11 +717,11 @@ private void dumpThread(PrintWriter output, ThreadInfo ti) { output.print(sb); } - private String getTestLogDirName() { + protected String getTestLogDirName() { if (testMethodName == null) { return testClassName; } - return testClassName + "_" + testMethodName; + return testClassName + File.separator + testMethodName; } public void setKillPoints(List killPoints) { diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index a93c54c52b82a..aafb3641fc1d7 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -38,6 +38,7 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES; @@ -56,23 +57,17 @@ public class ConfigNodeWrapper extends AbstractNodeWrapper { private final String defaultCommonPropertiesFile; public ConfigNodeWrapper( - boolean isSeed, - String targetCNs, - String testClassName, - String testMethodName, - int[] portList, - int clusterIndex, - boolean isMultiCluster, - long startTime) { + final boolean isSeed, + final String targetCNs, + final String testClassName, + final String testMethodName, + final int[] portList, + final int clusterIndex, + final boolean isMultiCluster, + final long startTime) { super(testClassName, testMethodName, portList, clusterIndex, isMultiCluster, startTime); this.consensusPort = portList[1]; this.isSeed = isSeed; - String seedConfigNodes; - if (isSeed) { - seedConfigNodes = getIpAndPortString(); - } else { - seedConfigNodes = targetCNs; - } this.defaultNodePropertiesFile = EnvUtils.getFilePathFromSysVar(DEFAULT_CONFIG_NODE_PROPERTIES, clusterIndex); this.defaultCommonPropertiesFile = @@ -82,7 +77,8 @@ public ConfigNodeWrapper( reloadMutableFields(); // initialize immutable properties - immutableNodeProperties.setProperty(IoTDBConstant.CN_SEED_CONFIG_NODE, seedConfigNodes); + immutableNodeProperties.setProperty( + IoTDBConstant.CN_SEED_CONFIG_NODE, isSeed ? getIpAndPortString() : targetCNs); immutableNodeProperties.setProperty(CN_SYSTEM_DIR, MppBaseConfig.NULL_VALUE); immutableNodeProperties.setProperty(CN_CONSENSUS_DIR, MppBaseConfig.NULL_VALUE); immutableNodeProperties.setProperty(CN_METRIC_IOTDB_REPORTER_HOST, MppBaseConfig.NULL_VALUE); @@ -128,7 +124,7 @@ public final String getId() { } @Override - protected void addStartCmdParams(List params) { + protected void addStartCmdParams(final List params) { final String workDir = getNodePath(); final String confDir = workDir + File.separator + "conf"; params.addAll( @@ -143,6 +139,11 @@ protected void addStartCmdParams(List params) { "-s")); } + @Override + String getNodeType() { + return "confignode"; + } + @Override protected void reloadMutableFields() { mutableCommonProperties.setProperty(CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS, SIMPLE_CONSENSUS); @@ -160,18 +161,19 @@ protected void reloadMutableFields() { IoTDBConstant.CN_CONSENSUS_PORT, String.valueOf(this.consensusPort)); mutableNodeProperties.setProperty( IoTDBConstant.CN_METRIC_PROMETHEUS_REPORTER_PORT, String.valueOf(super.getMetricPort())); + mutableNodeProperties.setProperty(CONFIG_NODE_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); } @Override protected void renameFile() { - String configNodeName = isSeed ? "SeedConfigNode" : "ConfigNode"; + final String configNodeName = isSeed ? "SeedConfigNode" : "ConfigNode"; // rename log file - File oldLogFile = + final File oldLogFile = new File(getLogDirPath() + File.separator + configNodeName + portList[0] + ".log"); oldLogFile.renameTo(new File(getLogDirPath() + File.separator + getId() + ".log")); // rename node dir - File oldNodeDir = + final File oldNodeDir = new File( System.getProperty(USER_DIR) + File.separator @@ -182,7 +184,7 @@ protected void renameFile() { oldNodeDir.renameTo(new File(getNodePath())); } - public void setConsensusPort(int consensusPort) { + public void setConsensusPort(final int consensusPort) { this.consensusPort = consensusPort; } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index 647bd9b9fa003..1d65adacf7e21 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -36,6 +36,9 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATANODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATA_NODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_PROPERTIES; @@ -55,15 +58,18 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAIN_CLASS_NAME; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAX_TSBLOCK_SIZE_IN_BYTES; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_DATA_PATH; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_HOST; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_PORT; import static org.apache.iotdb.it.env.cluster.ClusterConstant.PAGE_SIZE_IN_BYTE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_AIR_GAP_RECEIVER_PORT; import static org.apache.iotdb.it.env.cluster.ClusterConstant.REST_SERVICE_PORT; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET; import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.WAL_BUFFER_SIZE_IN_BYTE; public class DataNodeWrapper extends AbstractNodeWrapper { private int mppDataExchangePort; @@ -79,14 +85,16 @@ public class DataNodeWrapper extends AbstractNodeWrapper { private final String defaultCommonPropertiesFile; + private final int regionPerDataNode; + public DataNodeWrapper( - String seedConfigNode, - String testClassName, - String testMethodName, - int[] portList, - int clusterIndex, - boolean isMultiCluster, - long startTime) { + final String seedConfigNode, + final String testClassName, + final String testMethodName, + final int[] portList, + final int clusterIndex, + final boolean isMultiCluster, + final long startTime) { super(testClassName, testMethodName, portList, clusterIndex, isMultiCluster, startTime); this.internalAddress = super.getIp(); this.mppDataExchangePort = portList[1]; @@ -100,6 +108,7 @@ public DataNodeWrapper( EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_PROPERTIES, clusterIndex); this.defaultCommonPropertiesFile = EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_COMMON_PROPERTIES, clusterIndex); + this.regionPerDataNode = EnvUtils.getIntFromSysVar(DATA_REGION_PER_DATANODE, 0, clusterIndex); // Initialize mutable properties reloadMutableFields(); @@ -107,6 +116,8 @@ public DataNodeWrapper( // Override mqtt properties of super class immutableCommonProperties.setProperty(MQTT_HOST, super.getIp()); immutableCommonProperties.setProperty(MQTT_PORT, String.valueOf(this.mqttPort)); + immutableCommonProperties.setProperty( + MQTT_DATA_PATH, getNodePath() + File.separator + "mqttData"); immutableCommonProperties.setProperty( PIPE_AIR_GAP_RECEIVER_PORT, String.valueOf(this.pipeAirGapReceiverPort)); @@ -142,6 +153,14 @@ public String getSystemPropertiesPath() { return workDirFilePath("data/datanode/system/schema", IoTDBStartCheck.PROPERTIES_FILE_NAME); } + public String getDataDir() { + return getNodePath() + File.separator + "data"; + } + + public String getWalDir() { + return getDataDir() + File.separator + "datanode" + File.separator + "wal"; + } + @Override protected MppJVMConfig initVMConfig() { return MppJVMConfig.builder() @@ -149,6 +168,7 @@ protected MppJVMConfig initVMConfig() { .setMaxHeapSize(EnvUtils.getIntFromSysVar(DATANODE_MAX_HEAP_SIZE, 256, clusterIndex)) .setMaxDirectMemorySize( EnvUtils.getIntFromSysVar(DATANODE_MAX_DIRECT_MEMORY_SIZE, 256, clusterIndex)) + .setTimezone("Asia/Shanghai") .build(); } @@ -158,7 +178,7 @@ public final String getId() { } @Override - protected void addStartCmdParams(List params) { + protected void addStartCmdParams(final List params) { final String workDir = getNodePath(); final String confDir = workDir + File.separator + "conf"; params.addAll( @@ -172,6 +192,11 @@ protected void addStartCmdParams(List params) { "-s")); } + @Override + String getNodeType() { + return "datanode"; + } + @Override protected void reloadMutableFields() { mutableCommonProperties.setProperty(CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS, SIMPLE_CONSENSUS); @@ -203,27 +228,31 @@ protected void reloadMutableFields() { DN_DATA_REGION_CONSENSUS_PORT, String.valueOf(this.dataRegionConsensusPort)); mutableNodeProperties.setProperty( DN_SCHEMA_REGION_CONSENSUS_PORT, String.valueOf(this.schemaRegionConsensusPort)); + mutableNodeProperties.setProperty(WAL_BUFFER_SIZE_IN_BYTE, "16777216"); + mutableNodeProperties.setProperty(SCHEMA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); + mutableNodeProperties.setProperty(DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); + mutableNodeProperties.setProperty(DATA_REGION_PER_DATA_NODE, String.valueOf(regionPerDataNode)); } @Override public void renameFile() { // Rename log file - String oldLogFilePath = + final String oldLogFilePath = getLogDirPath() + File.separator + DATA_NODE_NAME + portList[0] + ".log"; - String newLogFilePath = getLogDirPath() + File.separator + getId() + ".log"; - File oldLogFile = new File(oldLogFilePath); + final String newLogFilePath = getLogDirPath() + File.separator + getId() + ".log"; + final File oldLogFile = new File(oldLogFilePath); oldLogFile.renameTo(new File(newLogFilePath)); // Rename node dir - String oldNodeDirPath = + final String oldNodeDirPath = System.getProperty(USER_DIR) + File.separator + TARGET + File.separator + DATA_NODE_NAME + portList[0]; - String newNodeDirPath = getNodePath(); - File oldNodeDir = new File(oldNodeDirPath); + final String newNodeDirPath = getNodePath(); + final File oldNodeDir = new File(oldNodeDirPath); oldNodeDir.renameTo(new File(newNodeDirPath)); } @@ -231,7 +260,7 @@ public int getMppDataExchangePort() { return mppDataExchangePort; } - public void setMppDataExchangePort(int mppDataExchangePort) { + public void setMppDataExchangePort(final int mppDataExchangePort) { this.mppDataExchangePort = mppDataExchangePort; } @@ -243,7 +272,7 @@ public int getInternalPort() { return internalPort; } - public void setInternalPort(int internalPort) { + public void setInternalPort(final int internalPort) { this.internalPort = internalPort; } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index dd93479ecf434..b189140cd081d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -54,6 +54,16 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptType(String encryptType) { + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + return this; + } + @Override public CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs) { return this; @@ -85,8 +95,7 @@ public CommonConfig setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompac } @Override - public CommonConfig setMaxInnerCompactionCandidateFileNum( - int maxInnerCompactionCandidateFileNum) { + public CommonConfig setInnerCompactionCandidateFileNum(int maxInnerCompactionCandidateFileNum) { return this; } @@ -106,7 +115,7 @@ public CommonConfig setPrimitiveArraySize(int primitiveArraySize) { } @Override - public CommonConfig setAvgSeriesPointNumberThreshold(int avgSeriesPointNumberThreshold) { + public CommonConfig setTargetChunkPointNum(int targetChunkPointNum) { return this; } @@ -131,6 +140,11 @@ public CommonConfig setDataRegionConsensusProtocolClass(String dataRegionConsens return this; } + @Override + public CommonConfig setIoTConsensusV2Mode(String ioTConsensusV2Mode) { + return this; + } + @Override public CommonConfig setSchemaRegionGroupExtensionPolicy(String schemaRegionGroupExtensionPolicy) { return this; @@ -166,6 +180,11 @@ public CommonConfig setTimePartitionInterval(long timePartitionInterval) { return this; } + @Override + public CommonConfig setTTLCheckInterval(long ttlCheckInterval) { + return this; + } + @Override public CommonConfig setTimePartitionOrigin(long timePartitionOrigin) { return this; @@ -202,6 +221,11 @@ public CommonConfig setEnableMQTTService(boolean enableMQTTService) { return this; } + @Override + public CommonConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { + return this; + } + @Override public CommonConfig setSchemaEngineMode(String schemaEngineMode) { return this; @@ -250,6 +274,11 @@ public CommonConfig setSeriesSlotNum(int seriesSlotNum) { return this; } + @Override + public CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass) { + return this; + } + @Override public CommonConfig setSchemaMemoryAllocate(String schemaMemoryAllocate) { return this; @@ -320,7 +349,7 @@ public CommonConfig setTagAttributeTotalSize(int tagAttributeTotalSize) { } @Override - public CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs) { + public CommonConfig setDnConnectionTimeoutMs(int connectionTimeoutMs) { return this; } @@ -340,4 +369,15 @@ public CommonConfig setPipeMetaSyncerInitialSyncDelayMinutes( public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes) { return this; } + + @Override + public CommonConfig setPipeConnectorRequestSliceThresholdBytes( + int pipeConnectorRequestSliceThresholdBytes) { + return this; + } + + @Override + public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java index 33a6bc48afd72..ae8645eff524e 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java @@ -28,4 +28,14 @@ public class RemoteConfigNodeConfig implements ConfigNodeConfig { public ConfigNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + + @Override + public ConfigNodeConfig setMetricPrometheusReporterUsername(String username) { + return this; + } + + @Override + public ConfigNodeConfig setMetricPrometheusReporterPassword(String password) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index fe89997bc41f8..c273daba49e21 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -28,6 +28,16 @@ public DataNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + @Override + public DataNodeConfig setMetricPrometheusReporterUsername(String username) { + return this; + } + + @Override + public DataNodeConfig setMetricPrometheusReporterPassword(String password) { + return this; + } + @Override public DataNodeConfig setEnableRestService(boolean enableRestService) { return this; @@ -43,4 +53,19 @@ public DataNodeConfig setLoadTsFileAnalyzeSchemaMemorySizeInBytes( long loadTsFileAnalyzeSchemaMemorySizeInBytes) { return this; } + + @Override + public DataNodeConfig setCompactionScheduleInterval(long compactionScheduleInterval) { + return this; + } + + @Override + public DataNodeConfig setEnableMQTTService(boolean enableMQTTService) { + return this; + } + + @Override + public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java index b8b29544520dc..c2308cfe10393 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java @@ -27,8 +27,10 @@ import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.rpc.thrift.IConfigNodeRPCService; import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.pool.ISessionPool; +import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; @@ -41,12 +43,15 @@ import org.apache.iotdb.jdbc.Constant; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.TableSessionBuilder; import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -72,7 +77,7 @@ public void initClusterEnvironment() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE root.init;"); - statement.execute("DELETE DATABASE root;"); + statement.execute("DELETE DATABASE root.init;"); } catch (Exception e) { e.printStackTrace(); throw new AssertionError(e.getMessage()); @@ -107,25 +112,29 @@ public ClusterConfig getConfig() { } @Override - public List getMetricPrometheusReporterContents() { + public List getMetricPrometheusReporterContents(String authHeader) { List result = new ArrayList<>(); result.add( getUrlContent( - Config.IOTDB_HTTP_URL_PREFIX + ip_addr + ":" + configNodeMetricPort + "/metrics")); + Config.IOTDB_HTTP_URL_PREFIX + ip_addr + ":" + configNodeMetricPort + "/metrics", + authHeader)); result.add( getUrlContent( - Config.IOTDB_HTTP_URL_PREFIX + ip_addr + ":" + dataNodeMetricPort + "/metrics")); + Config.IOTDB_HTTP_URL_PREFIX + ip_addr + ":" + dataNodeMetricPort + "/metrics", + authHeader)); return result; } @Override - public Connection getConnection(String username, String password) throws SQLException { + public Connection getConnection(String username, String password, String sqlDialect) + throws SQLException { Connection connection; try { Class.forName(Config.JDBC_DRIVER_NAME); connection = DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + ip_addr + ":" + port, this.user, this.password); + Config.IOTDB_URL_PREFIX + ip_addr + ":" + port, + BaseEnv.constructProperties(this.user, this.password, sqlDialect)); } catch (ClassNotFoundException e) { e.printStackTrace(); throw new AssertionError(); @@ -135,18 +144,19 @@ public Connection getConnection(String username, String password) throws SQLExce @Override public Connection getWriteOnlyConnectionWithSpecifiedDataNode( - DataNodeWrapper dataNode, String username, String password) { + DataNodeWrapper dataNode, String username, String password, String sqlDialect) { throw new UnsupportedOperationException(); } @Override public Connection getConnectionWithSpecifiedDataNode( DataNodeWrapper dataNode, String username, String password) throws SQLException { - return getConnection(username, password); + return getConnection(username, password, TREE_SQL_DIALECT); } @Override - public Connection getConnection(Constant.Version version, String username, String password) + public Connection getConnection( + Constant.Version version, String username, String password, String sqlDialect) throws SQLException { Connection connection; try { @@ -161,8 +171,7 @@ public Connection getConnection(Constant.Version version, String username, Strin + VERSION + "=" + version.toString(), - this.user, - this.password); + BaseEnv.constructProperties(this.user, this.password, sqlDialect)); } catch (ClassNotFoundException e) { e.printStackTrace(); throw new AssertionError(); @@ -170,6 +179,13 @@ public Connection getConnection(Constant.Version version, String username, Strin return connection; } + @Override + public Connection getConnection( + DataNodeWrapper dataNodeWrapper, String username, String password, String sqlDialect) + throws SQLException { + throw new UnsupportedOperationException(); + } + public void setTestMethodName(String testCaseName) { // Do nothing } @@ -201,33 +217,131 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() throws Client @Override public ISessionPool getSessionPool(int maxSize) { - return new SessionPool( - SessionConfig.DEFAULT_HOST, - SessionConfig.DEFAULT_PORT, - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - maxSize, - SessionConfig.DEFAULT_FETCH_SIZE, - 60_000, - false, - null, - SessionConfig.DEFAULT_REDIRECTION_MODE, - SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS, - SessionConfig.DEFAULT_VERSION, - SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY, - SessionConfig.DEFAULT_MAX_FRAME_SIZE); + return new SessionPool.Builder() + .host(SessionConfig.DEFAULT_HOST) + .port(SessionConfig.DEFAULT_PORT) + .user(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .maxSize(maxSize) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .waitToGetSessionTimeoutInMs(60_000) + .enableCompression(false) + .zoneId(null) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .connectionTimeoutInMs(SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS) + .version(SessionConfig.DEFAULT_VERSION) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .build(); + } + + @Override + public ITableSessionPool getTableSessionPool(int maxSize) { + return new TableSessionPoolBuilder() + .nodeUrls( + Collections.singletonList( + SessionConfig.DEFAULT_HOST + ":" + SessionConfig.DEFAULT_PORT)) + .user(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .maxSize(maxSize) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .waitToGetSessionTimeoutInMs(60_000) + .enableCompression(false) + .zoneId(null) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .connectionTimeoutInMs(SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .build(); + } + + @Override + public ITableSessionPool getTableSessionPool(int maxSize, String database) { + return new TableSessionPoolBuilder() + .nodeUrls( + Collections.singletonList( + SessionConfig.DEFAULT_HOST + ":" + SessionConfig.DEFAULT_PORT)) + .user(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .database(database) + .maxSize(maxSize) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .waitToGetSessionTimeoutInMs(60_000) + .enableCompression(false) + .zoneId(null) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .connectionTimeoutInMs(SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .build(); } @Override public ISession getSessionConnection() throws IoTDBConnectionException { - Session session = new Session(ip_addr, Integer.parseInt(port)); + Session session = new Session.Builder().host(ip_addr).port(Integer.parseInt(port)).build(); session.open(); return session; } + @Override + public ISession getSessionConnection(ZoneId zoneId) throws IoTDBConnectionException { + Session session = + new Session.Builder().host(ip_addr).port(Integer.parseInt(port)).zoneId(zoneId).build(); + session.open(); + return session; + } + + @Override + public ITableSession getTableSessionConnection() throws IoTDBConnectionException { + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(ip_addr + ":" + port)) + .build(); + } + + @Override + public ITableSession getTableSessionConnectionWithDB(String database) + throws IoTDBConnectionException { + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(ip_addr + ":" + port)) + .database(database) + .build(); + } + + @Override + public ITableSession getTableSessionConnection(List nodeUrls) + throws IoTDBConnectionException { + return new TableSessionBuilder() + .nodeUrls(nodeUrls) + .username(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .zoneId(null) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .build(); + } + + @Override + public ITableSession getTableSessionConnection(String userName, String password) + throws IoTDBConnectionException { + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(ip_addr + ":" + port)) + .username(userName) + .password(password) + .build(); + } + + @Override public ISession getSessionConnection(String userName, String password) throws IoTDBConnectionException { - Session session = new Session(ip_addr, Integer.parseInt(port), userName, password); + Session session = + new Session.Builder() + .host(ip_addr) + .port(Integer.parseInt(port)) + .username(userName) + .password(password) + .build(); session.open(); return session; } @@ -235,16 +349,17 @@ public ISession getSessionConnection(String userName, String password) @Override public ISession getSessionConnection(List nodeUrls) throws IoTDBConnectionException { Session session = - new Session( - Collections.singletonList(ip_addr + ":" + port), - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - SessionConfig.DEFAULT_FETCH_SIZE, - null, - SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY, - SessionConfig.DEFAULT_MAX_FRAME_SIZE, - SessionConfig.DEFAULT_REDIRECTION_MODE, - SessionConfig.DEFAULT_VERSION); + new Session.Builder() + .nodeUrls(Collections.singletonList(ip_addr + ":" + port)) + .username(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .fetchSize(SessionConfig.DEFAULT_FETCH_SIZE) + .zoneId(null) + .thriftDefaultBufferSize(SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .enableRedirection(SessionConfig.DEFAULT_REDIRECTION_MODE) + .version(SessionConfig.DEFAULT_VERSION) + .build(); session.open(); return session; } @@ -279,6 +394,11 @@ public void shutdownAllConfigNodes() { throw new UnsupportedOperationException(); } + @Override + public void shutdownForciblyAllConfigNodes() { + throw new UnsupportedOperationException(); + } + @Override public void ensureNodeStatus(List nodes, List targetStatus) { throw new UnsupportedOperationException(); @@ -344,6 +464,11 @@ public void shutdownAllDataNodes() { throw new UnsupportedOperationException(); } + @Override + public void shutdownForciblyAllDataNodes() { + throw new UnsupportedOperationException(); + } + @Override public int getMqttPort() { throw new UnsupportedOperationException(); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java index d30c0c36d5048..896745ad648ca 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java @@ -20,18 +20,16 @@ package org.apache.iotdb.it.utils; import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +48,7 @@ public class TsFileGenerator implements AutoCloseable { private final File tsFile; private final TsFileWriter writer; private final Map> device2TimeSet; - private final Map> device2MeasurementSchema; + private final Map> device2MeasurementSchema; private Random random; public TsFileGenerator(final File tsFile) throws IOException { @@ -70,7 +68,7 @@ public void resetRandom(final long seed) { } public void registerTimeseries( - final String path, final List measurementSchemaList) { + final String path, final List measurementSchemaList) { if (device2MeasurementSchema.containsKey(path)) { LOGGER.error("Register same device {}.", path); return; @@ -81,7 +79,7 @@ public void registerTimeseries( } public void registerAlignedTimeseries( - final String path, final List measurementSchemaList) + final String path, final List measurementSchemaList) throws WriteProcessException { if (device2MeasurementSchema.containsKey(path)) { LOGGER.error("Register same device {}.", path); @@ -95,39 +93,29 @@ public void registerAlignedTimeseries( public void generateData( final String device, final int number, final long timeGap, final boolean isAligned) throws IOException, WriteProcessException { - final List schemas = device2MeasurementSchema.get(device); + final List schemas = device2MeasurementSchema.get(device); final TreeSet timeSet = device2TimeSet.get(device); final Tablet tablet = new Tablet(device, schemas); - final long[] timestamps = tablet.timestamps; - final Object[] values = tablet.values; final long sensorNum = schemas.size(); long startTime = timeSet.isEmpty() ? 0L : timeSet.last(); for (long r = 0; r < number; r++) { - int row = tablet.rowSize++; + int row = tablet.getRowSize(); startTime += timeGap; - timestamps[row] = startTime; + tablet.addTimestamp(row, startTime); timeSet.add(startTime); for (int i = 0; i < sensorNum; i++) { - generateDataPoint(values[i], row, schemas.get(i)); + generateDataPoint(tablet, i, row, schemas.get(i)); } // write - if (tablet.rowSize == tablet.getMaxRowNumber()) { - if (!isAligned) { - writer.write(tablet); - } else { - writer.writeAligned(tablet); - } + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writer.writeTree(tablet); tablet.reset(); } } // write - if (tablet.rowSize != 0) { - if (!isAligned) { - writer.write(tablet); - } else { - writer.writeAligned(tablet); - } + if (tablet.getRowSize() != 0) { + writer.writeTree(tablet); tablet.reset(); } @@ -141,36 +129,30 @@ public void generateData( final boolean isAligned, final long startTimestamp) throws IOException, WriteProcessException { - final List schemas = device2MeasurementSchema.get(device); + final List schemas = device2MeasurementSchema.get(device); final TreeSet timeSet = device2TimeSet.get(device); final Tablet tablet = new Tablet(device, schemas); - final long[] timestamps = tablet.timestamps; - final Object[] values = tablet.values; final long sensorNum = schemas.size(); long startTime = startTimestamp; for (long r = 0; r < number; r++) { - final int row = tablet.rowSize++; + final int row = tablet.getRowSize(); startTime += timeGap; - timestamps[row] = startTime; + tablet.addTimestamp(row, startTime); timeSet.add(startTime); for (int i = 0; i < sensorNum; i++) { - generateDataPoint(values[i], row, schemas.get(i)); + generateDataPoint(tablet, i, row, schemas.get(i)); } // write - if (tablet.rowSize == tablet.getMaxRowNumber()) { - if (!isAligned) { - writer.write(tablet); - } else { - writer.writeAligned(tablet); - } + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writer.writeTree(tablet); tablet.reset(); } } // write - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { if (!isAligned) { - writer.write(tablet); + writer.writeTree(tablet); } else { writer.writeAligned(tablet); } @@ -180,97 +162,87 @@ public void generateData( LOGGER.info("Write {} points into device {}", number, device); } - private void generateDataPoint(final Object obj, final int row, final MeasurementSchema schema) { + private void generateDataPoint( + final Tablet tablet, final int column, final int row, final IMeasurementSchema schema) { switch (schema.getType()) { case INT32: - generateINT32(obj, row); + generateINT32(tablet, column, row); break; case DATE: - generateDATE(obj, row); + generateDATE(tablet, column, row); break; case INT64: case TIMESTAMP: - generateINT64(obj, row); + generateINT64(tablet, column, row); break; case FLOAT: - generateFLOAT(obj, row); + generateFLOAT(tablet, column, row); break; case DOUBLE: - generateDOUBLE(obj, row); + generateDOUBLE(tablet, column, row); break; case BOOLEAN: - generateBOOLEAN(obj, row); + generateBOOLEAN(tablet, column, row); break; case TEXT: case BLOB: case STRING: - generateTEXT(obj, row); + generateTEXT(tablet, column, row); break; default: LOGGER.error("Wrong data type {}.", schema.getType()); } } - private void generateINT32(final Object obj, final int row) { - final int[] ints = (int[]) obj; - ints[row] = random.nextInt(); + private void generateINT32(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextInt()); } - private void generateDATE(final Object obj, final int row) { - final LocalDate[] dates = (LocalDate[]) obj; - dates[row] = - LocalDate.of(1000 + random.nextInt(9000), 1 + random.nextInt(12), 1 + random.nextInt(28)); + private void generateDATE(final Tablet tablet, final int column, final int row) { + tablet.addValue( + row, + column, + LocalDate.of(1000 + random.nextInt(9000), 1 + random.nextInt(12), 1 + random.nextInt(28))); } - private void generateINT64(final Object obj, final int row) { - final long[] longs = (long[]) obj; - longs[row] = random.nextLong(); + private void generateINT64(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextLong()); } - private void generateFLOAT(final Object obj, final int row) { - final float[] floats = (float[]) obj; - floats[row] = random.nextFloat(); + private void generateFLOAT(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextFloat()); } - private void generateDOUBLE(final Object obj, final int row) { - final double[] doubles = (double[]) obj; - doubles[row] = random.nextDouble(); + private void generateDOUBLE(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextDouble()); } - private void generateBOOLEAN(final Object obj, final int row) { - final boolean[] booleans = (boolean[]) obj; - booleans[row] = random.nextBoolean(); + private void generateBOOLEAN(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextBoolean()); } - private void generateTEXT(final Object obj, final int row) { - final Binary[] binaries = (Binary[]) obj; - binaries[row] = - new Binary(String.format("test point %d", random.nextInt()), TSFileConfig.STRING_CHARSET); + private void generateTEXT(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, String.format("test point %d", random.nextInt())); } public void generateDeletion(final String device, final int number) throws IOException, IllegalPathException { try (final ModificationFile modificationFile = - new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) { - writer.flushAllChunkGroups(); + new ModificationFile(ModificationFile.getExclusiveMods(tsFile), false)) { + writer.flush(); final TreeSet timeSet = device2TimeSet.get(device); if (timeSet.isEmpty()) { return; } - final long fileOffset = tsFile.length(); final long maxTime = timeSet.last() - 1; for (int i = 0; i < number; i++) { final int endTime = random.nextInt((int) (maxTime)) + 1; final int startTime = random.nextInt(endTime); - for (final MeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) { - final Deletion deletion = - new Deletion( - new PartialPath( - device - + TsFileConstant.PATH_SEPARATOR - + measurementSchema.getMeasurementId()), - fileOffset, + for (final IMeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) { + final ModEntry deletion = + new TreeDeletionEntry( + new MeasurementPath(device, measurementSchema.getMeasurementName()), startTime, endTime); modificationFile.write(deletion); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java new file mode 100644 index 0000000000000..6d86a5079f532 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.it.utils; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Collectors; + +public class TsFileTableGenerator implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileTableGenerator.class); + + private final File tsFile; + private final TsFileWriter writer; + private final Map> table2TimeSet; + private final Map> table2MeasurementSchema; + private final Map> table2ColumnCategory; + private Random random; + + public TsFileTableGenerator(final File tsFile) throws IOException { + this.tsFile = tsFile; + this.writer = new TsFileWriter(tsFile); + this.table2TimeSet = new HashMap<>(); + this.table2MeasurementSchema = new HashMap<>(); + this.table2ColumnCategory = new HashMap<>(); + this.random = new Random(); + } + + public void registerTable( + final String tableName, + final List columnSchemasList, + final List columnCategoryList) { + if (table2MeasurementSchema.containsKey(tableName)) { + LOGGER.warn("Table {} already exists", tableName); + return; + } + + writer.registerTableSchema(new TableSchema(tableName, columnSchemasList, columnCategoryList)); + table2TimeSet.put(tableName, new TreeSet<>()); + table2MeasurementSchema.put(tableName, columnSchemasList); + table2ColumnCategory.put(tableName, columnCategoryList); + } + + public void generateData(final String tableName, final int number, final long timeGap) + throws IOException, WriteProcessException { + final List schemas = table2MeasurementSchema.get(tableName); + final List columnNameList = + schemas.stream().map(IMeasurementSchema::getMeasurementName).collect(Collectors.toList()); + final List dataTypeList = + schemas.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()); + final List columnCategoryList = table2ColumnCategory.get(tableName); + final TreeSet timeSet = table2TimeSet.get(tableName); + final Tablet tablet = new Tablet(tableName, columnNameList, dataTypeList, columnCategoryList); + final Object[] values = tablet.getValues(); + final long sensorNum = schemas.size(); + long startTime = timeSet.isEmpty() ? 0L : timeSet.last(); + + for (long r = 0; r < number; r++) { + final int row = tablet.getRowSize(); + startTime += timeGap; + tablet.addTimestamp(row, startTime); + timeSet.add(startTime); + for (int i = 0; i < sensorNum; i++) { + generateDataPoint(tablet, i, row, schemas.get(i)); + } + // write + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writer.writeTable(tablet); + tablet.reset(); + } + } + // write + if (tablet.getRowSize() != 0) { + writer.writeTable(tablet); + tablet.reset(); + } + + LOGGER.info("Write {} points into table {}", number, tableName); + } + + private void generateDataPoint( + final Tablet tablet, final int column, final int row, final IMeasurementSchema schema) { + switch (schema.getType()) { + case INT32: + generateINT32(tablet, column, row); + break; + case DATE: + generateDATE(tablet, column, row); + break; + case INT64: + case TIMESTAMP: + generateINT64(tablet, column, row); + break; + case FLOAT: + generateFLOAT(tablet, column, row); + break; + case DOUBLE: + generateDOUBLE(tablet, column, row); + break; + case BOOLEAN: + generateBOOLEAN(tablet, column, row); + break; + case TEXT: + case BLOB: + case STRING: + generateTEXT(tablet, column, row); + break; + default: + LOGGER.error("Wrong data type {}.", schema.getType()); + } + } + + private void generateINT32(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextInt()); + } + + private void generateDATE(final Tablet tablet, final int column, final int row) { + tablet.addValue( + row, + column, + LocalDate.of(1000 + random.nextInt(9000), 1 + random.nextInt(12), 1 + random.nextInt(28))); + } + + private void generateINT64(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextLong()); + } + + private void generateFLOAT(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextFloat()); + } + + private void generateDOUBLE(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextDouble()); + } + + private void generateBOOLEAN(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, random.nextBoolean()); + } + + private void generateTEXT(final Tablet tablet, final int column, final int row) { + tablet.addValue(row, column, String.format("test point %d", random.nextInt())); + } + + public long getTotalNumber() { + return table2TimeSet.entrySet().stream() + .mapToInt( + entry -> entry.getValue().size() * table2MeasurementSchema.get(entry.getKey()).size()) + .sum(); + } + + @Override + public void close() throws Exception { + writer.close(); + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java new file mode 100644 index 0000000000000..ab3c458cd79c1 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface AIClusterIT {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/ManualIT.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/ManualIT.java new file mode 100644 index 0000000000000..78eda3b8d1b1f --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/ManualIT.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface ManualIT {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java deleted file mode 100644 index 25000e3012ae3..0000000000000 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.itbase.category; - -public interface MultiClusterIT2AutoCreateSchema {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java new file mode 100644 index 0000000000000..07eb001fcde30 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTableManualBasic {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java new file mode 100644 index 0000000000000..701401c268545 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTableManualEnhanced {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java new file mode 100644 index 0000000000000..477a606584b67 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTreeAutoBasic {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java new file mode 100644 index 0000000000000..7ce45f738104f --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTreeAutoEnhanced {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java new file mode 100644 index 0000000000000..b9e3e2f99d122 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTreeManual {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java deleted file mode 100644 index 2fb36dd22920a..0000000000000 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.itbase.category; - -public interface MultiClusterIT2ManualCreateSchema {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java deleted file mode 100644 index 34d870e4d6d1d..0000000000000 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.itbase.category; - -public interface MultiClusterIT2Subscription {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableArchVerification.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableArchVerification.java new file mode 100644 index 0000000000000..dff8f4b0e3658 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableArchVerification.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionTableArchVerification {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableRegressionConsumer.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableRegressionConsumer.java new file mode 100644 index 0000000000000..88cb4adbab220 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableRegressionConsumer.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionTableRegressionConsumer {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableRegressionMisc.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableRegressionMisc.java new file mode 100644 index 0000000000000..fafa9915aaed0 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTableRegressionMisc.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionTableRegressionMisc {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeArchVerification.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeArchVerification.java new file mode 100644 index 0000000000000..5c3531d5281fa --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeArchVerification.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionTreeArchVerification {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeRegressionConsumer.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeRegressionConsumer.java new file mode 100644 index 0000000000000..71b1f1f0869dc --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeRegressionConsumer.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionTreeRegressionConsumer {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeRegressionMisc.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeRegressionMisc.java new file mode 100644 index 0000000000000..173f4c8044ab6 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionTreeRegressionMisc.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionTreeRegressionMisc {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/TableClusterIT.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/TableClusterIT.java new file mode 100644 index 0000000000000..0ba2c4bafc863 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/TableClusterIT.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface TableClusterIT {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/TableLocalStandaloneIT.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/TableLocalStandaloneIT.java new file mode 100644 index 0000000000000..c6b39ad0dd52d --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/TableLocalStandaloneIT.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface TableLocalStandaloneIT {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/constant/TestConstant.java b/integration-test/src/main/java/org/apache/iotdb/itbase/constant/TestConstant.java index 18fd6fa64d96e..8a9d11516c5ae 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/constant/TestConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/constant/TestConstant.java @@ -19,7 +19,7 @@ package org.apache.iotdb.itbase.constant; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.tsfile.utils.FilePathUtils; import org.apache.tsfile.write.record.TSRecord; @@ -51,7 +51,7 @@ public class TestConstant { public static boolean testFlag = true; public static String[] stringValue = new String[] {"A", "B", "C", "D", "E"}; public static String[] booleanValue = new String[] {"true", "false"}; - public static final String TIMESEIRES_STR = ColumnHeaderConstant.TIMESERIES; + public static final String TIMESERIES_STR = ColumnHeaderConstant.TIMESERIES; public static final String VALUE_STR = ColumnHeaderConstant.VALUE; public static final String DATA_TYPE_STR = ColumnHeaderConstant.DATATYPE; public static final String FUNCTION_TYPE_NATIVE = "native"; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/constant/UDFTestConstant.java b/integration-test/src/main/java/org/apache/iotdb/itbase/constant/UDFTestConstant.java index 8114f5961016a..2732a97cccc87 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/constant/UDFTestConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/constant/UDFTestConstant.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.itbase.constant; public class UDFTestConstant { diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java index 3b9b27de8517c..19dcc77dfdebe 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java @@ -23,14 +23,19 @@ import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.rpc.thrift.IConfigNodeRPCService; import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.pool.ISessionPool; +import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.Constant; import org.apache.iotdb.rpc.IoTDBConnectionException; +import reactor.util.annotation.Nullable; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -39,11 +44,17 @@ import java.net.URL; import java.sql.Connection; import java.sql.SQLException; +import java.time.ZoneId; import java.util.List; import java.util.Optional; +import java.util.Properties; public interface BaseEnv { + String TREE_SQL_DIALECT = "tree"; + + String TABLE_SQL_DIALECT = "table"; + /** Init a cluster with default number of ConfigNodes and DataNodes. */ void initClusterEnvironment(); @@ -70,11 +81,14 @@ public interface BaseEnv { /** Return the {@link ClusterConfig} for developers to set values before test. */ ClusterConfig getConfig(); - default String getUrlContent(String urlStr) { + default String getUrlContent(String urlStr, @Nullable String authHeader) { StringBuilder sb = new StringBuilder(); try { URL url = new URL(urlStr); HttpURLConnection httpConnection = (HttpURLConnection) url.openConnection(); + if (authHeader != null) { + httpConnection.setRequestProperty("Authorization", authHeader); + } if (httpConnection.getResponseCode() == HttpURLConnection.HTTP_OK) { InputStream in = httpConnection.getInputStream(); InputStreamReader isr = new InputStreamReader(in); @@ -96,30 +110,68 @@ default String getUrlContent(String urlStr) { } /** Return the content of prometheus */ - List getMetricPrometheusReporterContents(); + List getMetricPrometheusReporterContents(String authHeader); default Connection getConnection() throws SQLException { - return getConnection(SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + return getConnection( + SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, TREE_SQL_DIALECT); + } + + default Connection getTableConnection() throws SQLException { + return getConnection( + SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, TABLE_SQL_DIALECT); + } + + default Connection getConnection(String sqlDialect) throws SQLException { + return getConnection(SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); } default Connection getConnection(Constant.Version version) throws SQLException { - return getConnection(version, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + return getConnection( + version, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, TREE_SQL_DIALECT); + } + + default Connection getConnection(Constant.Version version, String sqlDialect) + throws SQLException { + return getConnection( + version, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); } - Connection getConnection(Constant.Version version, String username, String password) + default Connection getConnection(Constant.Version version, String username, String password) + throws SQLException { + return getConnection(version, username, password, TREE_SQL_DIALECT); + } + + Connection getConnection( + Constant.Version version, String username, String password, String sqlDialect) + throws SQLException; + + Connection getConnection( + DataNodeWrapper dataNodeWrapper, String username, String password, String sqlDialect) throws SQLException; - Connection getConnection(String username, String password) throws SQLException; + default Connection getConnection(String username, String password) throws SQLException { + return getConnection(username, password, TREE_SQL_DIALECT); + } + + Connection getConnection(String username, String password, String sqlDialect) throws SQLException; default Connection getWriteOnlyConnectionWithSpecifiedDataNode(DataNodeWrapper dataNode) throws SQLException { return getWriteOnlyConnectionWithSpecifiedDataNode( - dataNode, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + dataNode, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, TREE_SQL_DIALECT); + } + + default Connection getWriteOnlyConnectionWithSpecifiedDataNode( + DataNodeWrapper dataNode, String sqlDialect) throws SQLException { + return getWriteOnlyConnectionWithSpecifiedDataNode( + dataNode, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); } // This is useful when you shut down a dataNode. Connection getWriteOnlyConnectionWithSpecifiedDataNode( - DataNodeWrapper dataNode, String username, String password) throws SQLException; + DataNodeWrapper dataNode, String username, String password, String sqlDialect) + throws SQLException; default Connection getConnectionWithSpecifiedDataNode(DataNodeWrapper dataNode) throws SQLException { @@ -145,12 +197,27 @@ IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() ISessionPool getSessionPool(int maxSize); + ITableSessionPool getTableSessionPool(int maxSize); + + ITableSessionPool getTableSessionPool(int maxSize, String database); + ISession getSessionConnection() throws IoTDBConnectionException; + ISession getSessionConnection(ZoneId zoneId) throws IoTDBConnectionException; + ISession getSessionConnection(String userName, String password) throws IoTDBConnectionException; ISession getSessionConnection(List nodeUrls) throws IoTDBConnectionException; + ITableSession getTableSessionConnection() throws IoTDBConnectionException; + + ITableSession getTableSessionConnectionWithDB(String database) throws IoTDBConnectionException; + + ITableSession getTableSessionConnection(List nodeUrls) throws IoTDBConnectionException; + + ITableSession getTableSessionConnection(String userName, String password) + throws IoTDBConnectionException; + /** * Get the index of the first dataNode with a SchemaRegion leader. * @@ -181,6 +248,8 @@ default IConfigNodeRPCService.Iface getConfigNodeConnection(int index) throws Ex /** Shutdown all existed ConfigNodes. */ void shutdownAllConfigNodes(); + void shutdownForciblyAllConfigNodes(); + /** * Ensure all the nodes being in the corresponding status. * @@ -245,6 +314,9 @@ void ensureNodeStatus(List nodes, List targetStatus /** Shutdown all existed DataNodes. */ void shutdownAllDataNodes(); + /** Shutdown forcibly all existed DataNodes. */ + void shutdownForciblyAllDataNodes(); + int getMqttPort(); String getIP(); @@ -262,4 +334,19 @@ void ensureNodeStatus(List nodes, List targetStatus void registerConfigNodeKillPoints(List killPoints); void registerDataNodeKillPoints(List killPoints); + + static Properties constructProperties(String username, String password, String sqlDialect) { + Properties info = new Properties(); + + if (username != null) { + info.put("user", username); + } + if (password != null) { + info.put("password", password); + } + if (sqlDialect != null) { + info.put(Config.SQL_DIALECT, sqlDialect); + } + return info; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index 840b9effde401..afb356b846aa3 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -36,6 +36,10 @@ public interface CommonConfig { CommonConfig setCompressor(String compressor); + CommonConfig setEncryptType(String encryptType); + + CommonConfig setEncryptKeyPath(String encryptKeyPath); + CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs); CommonConfig setUdfMemoryBudgetInMB(float udfCollectorMemoryBudgetInMB); @@ -48,7 +52,7 @@ public interface CommonConfig { CommonConfig setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompaction); - CommonConfig setMaxInnerCompactionCandidateFileNum(int maxInnerCompactionCandidateFileNum); + CommonConfig setInnerCompactionCandidateFileNum(int maxInnerCompactionCandidateFileNum); CommonConfig setAutoCreateSchemaEnabled(boolean enableAutoCreateSchema); @@ -56,7 +60,7 @@ public interface CommonConfig { CommonConfig setPrimitiveArraySize(int primitiveArraySize); - CommonConfig setAvgSeriesPointNumberThreshold(int avgSeriesPointNumberThreshold); + CommonConfig setTargetChunkPointNum(int targetChunkPointNum); CommonConfig setMaxTsBlockLineNumber(int maxTsBlockLineNumber); @@ -66,6 +70,8 @@ public interface CommonConfig { CommonConfig setDataRegionConsensusProtocolClass(String dataRegionConsensusProtocolClass); + CommonConfig setIoTConsensusV2Mode(String ioTConsensusV2Mode); + CommonConfig setSchemaRegionGroupExtensionPolicy(String schemaRegionGroupExtensionPolicy); CommonConfig setDefaultSchemaRegionGroupNumPerDatabase(int schemaRegionGroupPerDatabase); @@ -80,6 +86,8 @@ public interface CommonConfig { CommonConfig setTimePartitionInterval(long timePartitionInterval); + CommonConfig setTTLCheckInterval(long ttlCheckInterval); + CommonConfig setTimePartitionOrigin(long timePartitionOrigin); CommonConfig setTimestampPrecision(String timestampPrecision); @@ -94,6 +102,8 @@ public interface CommonConfig { CommonConfig setEnableMQTTService(boolean enableMQTTService); + CommonConfig setMqttPayloadFormatter(String mqttPayloadFormatter); + CommonConfig setSchemaEngineMode(String schemaEngineMode); CommonConfig setSelectIntoInsertTabletPlanRowLimit(int selectIntoInsertTabletPlanRowLimit); @@ -114,6 +124,8 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus( CommonConfig setSeriesSlotNum(int seriesSlotNum); + CommonConfig setSeriesPartitionExecutorClass(String seriesPartitionExecutorClass); + CommonConfig setSchemaMemoryAllocate(String schemaMemoryAllocate); CommonConfig setWriteMemoryProportion(String writeMemoryProportion); @@ -142,7 +154,7 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus( CommonConfig setTagAttributeTotalSize(int tagAttributeTotalSize); - CommonConfig setCnConnectionTimeoutMs(int connectionTimeoutMs); + CommonConfig setDnConnectionTimeoutMs(int connectionTimeoutMs); CommonConfig setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( int pipeHeartbeatIntervalSecondsForCollectingPipeMeta); @@ -150,4 +162,13 @@ CommonConfig setPipeHeartbeatIntervalSecondsForCollectingPipeMeta( CommonConfig setPipeMetaSyncerInitialSyncDelayMinutes(long pipeMetaSyncerInitialSyncDelayMinutes); CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes); + + CommonConfig setPipeConnectorRequestSliceThresholdBytes( + int pipeConnectorRequestSliceThresholdBytes); + + CommonConfig setQueryMemoryProportion(String queryMemoryProportion); + + default CommonConfig setDefaultStorageGroupLevel(int defaultStorageGroupLevel) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java index bf7179ef70289..65a5a3271fc19 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java @@ -23,5 +23,10 @@ /** This interface is used to handle properties in iotdb-confignode.properties. */ public interface ConfigNodeConfig { + ConfigNodeConfig setMetricReporterType(List metricReporterTypes); + + ConfigNodeConfig setMetricPrometheusReporterUsername(String username); + + ConfigNodeConfig setMetricPrometheusReporterPassword(String password); } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index 2887b0a987189..b8c44423bf835 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -25,10 +25,20 @@ public interface DataNodeConfig { DataNodeConfig setMetricReporterType(List metricReporterTypes); + DataNodeConfig setMetricPrometheusReporterUsername(String username); + + DataNodeConfig setMetricPrometheusReporterPassword(String password); + DataNodeConfig setEnableRestService(boolean enableRestService); DataNodeConfig setConnectionTimeoutInMS(int connectionTimeoutInMS); DataNodeConfig setLoadTsFileAnalyzeSchemaMemorySizeInBytes( long loadTsFileAnalyzeSchemaMemorySizeInBytes); + + DataNodeConfig setCompactionScheduleInterval(long compactionScheduleInterval); + + DataNodeConfig setEnableMQTTService(boolean enableMQTTService); + + DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter); } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java index c6b139a3eeb26..f5dc5289d1268 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java @@ -31,6 +31,9 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.iotdb.rpc.RpcUtils.isSetSqlDialect; +import static org.apache.iotdb.rpc.RpcUtils.isUseDatabase; + /** The implementation of {@link ClusterTestStatement} in cluster test. */ public class ClusterTestStatement implements Statement { @@ -187,7 +190,16 @@ public void setCursorName(String name) { @Override public boolean execute(String sql) throws SQLException { - return writeStatement.execute(sql); + sql = sql.trim(); + boolean result = writeStatement.execute(sql); + // if 'use XXXX' or 'set sql_dialect', sendRequest to all statements + if (isUseDatabase(sql) || isSetSqlDialect(sql)) { + for (Statement readStatement : readStatements) { + boolean tmp = readStatement.execute(sql); + result = result && tmp; + } + } + return result; } @Override diff --git a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java new file mode 100644 index 0000000000000..07d29c0d224ed --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.ainode.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.AIClusterIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({AIClusterIT.class}) +public class AINodeBasicIT { + static final String MODEL_PATH = + System.getProperty("user.dir") + + File.separator + + "src" + + File.separator + + "test" + + File.separator + + "resources" + + File.separator + + "ainode-example"; + + static String[] sqls = + new String[] { + "set configuration \"trusted_uri_pattern\"='.*'", + "create model identity using uri \"" + MODEL_PATH + "\"", + "CREATE DATABASE root.AI.data", + "CREATE TIMESERIES root.AI.data.s0 WITH DATATYPE=FLOAT, ENCODING=RLE", + "CREATE TIMESERIES root.AI.data.s1 WITH DATATYPE=FLOAT, ENCODING=RLE", + "CREATE TIMESERIES root.AI.data.s2 WITH DATATYPE=FLOAT, ENCODING=RLE", + "CREATE TIMESERIES root.AI.data.s3 WITH DATATYPE=DOUBLE, ENCODING=RLE", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(1,1.0,2.0,3.0,4.0)", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(2,2.0,3.0,4.0,5.0)", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(3,3.0,4.0,5.0,6.0)", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(4,4.0,5.0,6.0,7.0)", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(5,5.0,6.0,7.0,8.0)", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(6,6.0,7.0,8.0,9.0)", + "insert into root.AI.data(timestamp,s0,s1,s2,s3) values(7,7.0,8.0,9.0,10.0)", + }; + + @BeforeClass + public static void setUp() throws Exception { + // Init 1C1D1M cluster environment + EnvFactory.getEnv().initClusterEnvironment(1, 1); + prepareData(sqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void checkHeader(ResultSetMetaData resultSetMetaData, String title) + throws SQLException { + String[] headers = title.split(","); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(headers[i - 1], resultSetMetaData.getColumnName(i)); + } + } + + private void errorTest(String sql, String errorMessage) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet ignored = statement.executeQuery(sql)) { + fail("There should be an exception"); + } + } catch (SQLException e) { + assertEquals(errorMessage, e.getMessage()); + } + } + + @Test + public void aiNodeConnectionTest() { + String sql = "SHOW AINODES"; + String title = "NodeID,Status,RpcAddress,RpcPort"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader(resultSetMetaData, title); + int count = 0; + while (resultSet.next()) { + assertEquals("2", resultSet.getString(1)); + assertEquals("Running", resultSet.getString(2)); + count++; + } + assertEquals(1, count); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void ModelOperationTest() { + String registerSql = "create model operationTest using uri \"" + MODEL_PATH + "\""; + String showSql = "SHOW MODELS operationTest"; + String dropSql = "DROP MODEL operationTest"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute(registerSql); + boolean loading = true; + int count = 0; + while (loading) { + try (ResultSet resultSet = statement.executeQuery(showSql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader(resultSetMetaData, "ModelId,ModelType,State,Configs,Notes"); + while (resultSet.next()) { + String modelName = resultSet.getString(1); + String modelType = resultSet.getString(2); + String status = resultSet.getString(3); + + assertEquals("operationTest", modelName); + assertEquals("USER_DEFINED", modelType); + if (status.equals("ACTIVE")) { + loading = false; + count++; + } else if (status.equals("LOADING")) { + break; + } else { + fail("Unexpected status of model: " + status); + } + } + } + } + assertEquals(1, count); + statement.execute(dropSql); + try (ResultSet resultSet = statement.executeQuery(showSql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader(resultSetMetaData, "ModelId,ModelType,State,Configs,Notes"); + count = 0; + while (resultSet.next()) { + count++; + } + assertEquals(0, count); + } + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void callInferenceTest() { + String sql = + "CALL INFERENCE(identity, \"select s0,s1,s2 from root.AI.data\", generateTime=true)"; + String sql2 = "CALL INFERENCE(identity, \"select s2,s0,s1 from root.AI.data\")"; + String sql3 = + "CALL INFERENCE(_NaiveForecaster, \"select s0 from root.AI.data\", predict_length=3, generateTime=true)"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader(resultSetMetaData, "Time,output0,output1,output2"); + int count = 0; + while (resultSet.next()) { + float s0 = resultSet.getFloat(2); + float s1 = resultSet.getFloat(3); + float s2 = resultSet.getFloat(4); + + assertEquals(s0, count + 1.0, 0.0001); + assertEquals(s1, count + 2.0, 0.0001); + assertEquals(s2, count + 3.0, 0.0001); + count++; + } + assertEquals(7, count); + } + + try (ResultSet resultSet = statement.executeQuery(sql2)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader(resultSetMetaData, "output0,output1,output2"); + int count = 0; + while (resultSet.next()) { + float s2 = resultSet.getFloat(1); + float s0 = resultSet.getFloat(2); + float s1 = resultSet.getFloat(3); + + assertEquals(s0, count + 1.0, 0.0001); + assertEquals(s1, count + 2.0, 0.0001); + assertEquals(s2, count + 3.0, 0.0001); + count++; + } + assertEquals(7, count); + } + + try (ResultSet resultSet = statement.executeQuery(sql3)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader(resultSetMetaData, "Time,output0,output1,output2"); + int count = 0; + while (resultSet.next()) { + count++; + } + assertEquals(3, count); + } + + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void errorTest() { + String sql = + "CALL INFERENCE(notFound404, \"select s0,s1,s2 from root.AI.data\", window=head(5))"; + errorTest(sql, "1505: model [notFound404] has not been created."); + sql = "CALL INFERENCE(identity, \"select s0,s1,s2 from root.AI.data\", window=head(2))"; + errorTest(sql, "701: Window output 2 is not equal to input size of model 7"); + sql = "CALL INFERENCE(identity, \"select s0,s1,s2 from root.AI.data limit 5\")"; + errorTest( + sql, + "301: The number of rows 5 in the input data does not match the model input 7. Try to use LIMIT in SQL or WINDOW in CALL INFERENCE"); + sql = "CREATE MODEL 中文 USING URI \"" + MODEL_PATH + "\""; + errorTest(sql, "701: ModelName can only contain letters, numbers, and underscores"); + sql = "DROP MODEL _GaussianHMM"; + errorTest(sql, "1502: Built-in model _GaussianHMM can't be removed"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/cli/it/AbstractScript.java b/integration-test/src/test/java/org/apache/iotdb/cli/it/AbstractScript.java deleted file mode 100644 index 53f51cb81a6ea..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/cli/it/AbstractScript.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.cli.it; - -import org.apache.iotdb.it.framework.IoTDBTestRunner; - -import org.apache.thrift.annotation.Nullable; -import org.junit.runner.RunWith; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -public abstract class AbstractScript { - - protected void testOutput(ProcessBuilder builder, @Nullable String[] output, int statusCode) - throws IOException { - builder.redirectErrorStream(true); - Process p = builder.start(); - BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line; - List outputList = new ArrayList<>(); - while (true) { - line = r.readLine(); - if (line == null) { - break; - } else { - outputList.add(line); - } - } - r.close(); - p.destroy(); - - System.out.println("Process output:"); - for (String s : outputList) { - System.out.println(s); - } - - if (output != null) { - for (int i = 0; i < output.length; i++) { - assertTrue( - outputList.get(outputList.size() - 1 - i).contains(output[output.length - 1 - i])); - } - } - while (p.isAlive()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - e.printStackTrace(); - fail(); - } - } - assertEquals(statusCode, p.exitValue()); - } - - protected abstract void testOnWindows() throws IOException; - - protected abstract void testOnUnix() throws IOException; -} diff --git a/integration-test/src/test/java/org/apache/iotdb/cli/it/AbstractScriptIT.java b/integration-test/src/test/java/org/apache/iotdb/cli/it/AbstractScriptIT.java new file mode 100644 index 0000000000000..a0a25a2e12e0d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/cli/it/AbstractScriptIT.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.cli.it; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; + +import org.apache.thrift.annotation.Nullable; +import org.junit.runner.RunWith; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +public abstract class AbstractScriptIT { + + protected void testOutput(ProcessBuilder builder, @Nullable String[] output, int statusCode) + throws IOException { + builder.redirectErrorStream(true); + Process p = builder.start(); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + String line; + List outputList = new ArrayList<>(); + while (true) { + line = r.readLine(); + if (line == null) { + break; + } else { + outputList.add(line); + } + } + r.close(); + p.destroy(); + + System.out.println("Process output:"); + for (String s : outputList) { + System.out.println(s); + } + + if (output != null) { + for (int i = 0; i < output.length; i++) { + assertTrue( + outputList.get(outputList.size() - 1 - i).contains(output[output.length - 1 - i])); + } + } + while (p.isAlive()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + } + assertEquals(statusCode, p.exitValue()); + } + + protected abstract void testOnWindows() throws IOException; + + protected abstract void testOnUnix() throws IOException; +} diff --git a/integration-test/src/test/java/org/apache/iotdb/cli/it/StartClientScriptIT.java b/integration-test/src/test/java/org/apache/iotdb/cli/it/StartClientScriptIT.java index 317ec218fc617..bd1a7d8820747 100644 --- a/integration-test/src/test/java/org/apache/iotdb/cli/it/StartClientScriptIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/cli/it/StartClientScriptIT.java @@ -35,7 +35,7 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class StartClientScriptIT extends AbstractScript { +public class StartClientScriptIT extends AbstractScriptIT { private static String ip; @@ -77,13 +77,13 @@ public void test() throws IOException { protected void testOnWindows() throws IOException { final String[] output = { - "Error: Connection Error, please check whether the network is available or the server has started. Host is 127.0.0.1, port is 6668." + "Error: Connection Error, please check whether the network is available or the server has started." }; ProcessBuilder builder = new ProcessBuilder( "cmd.exe", "/c", - sbinPath + File.separator + "start-cli.bat", + sbinPath + File.separator + "windows" + File.separator + "start-cli.bat", "-h", ip, "-p", @@ -103,7 +103,7 @@ protected void testOnWindows() throws IOException { new ProcessBuilder( "cmd.exe", "/c", - sbinPath + File.separator + "start-cli.bat", + sbinPath + File.separator + "windows" + File.separator + "start-cli.bat", "-h", ip, "-p", @@ -121,7 +121,7 @@ protected void testOnWindows() throws IOException { protected void testOnUnix() throws IOException { final String[] output = { - "Error: Connection Error, please check whether the network is available or the server has started. Host is 127.0.0.1, port is 6668." + "Error: Connection Error, please check whether the network is available or the server has started." }; ProcessBuilder builder = new ProcessBuilder( diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java index af7cb37669d18..54599cfc809ee 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java @@ -19,6 +19,8 @@ package org.apache.iotdb.confignode.it; +import org.apache.iotdb.common.rpc.thrift.FunctionType; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; @@ -39,6 +41,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TGetTriggerTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetUDFTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetUdfTableReq; import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; @@ -71,9 +74,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils.generatePatternTreeBuffer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; @RunWith(IoTDBTestRunner.class) @Category({ClusterIT.class}) @@ -183,7 +188,7 @@ public void testPartitionInfoSnapshot() throws Exception { } assertTriggerInformation(createTriggerReqs, client.getTriggerTable()); - assertUDFInformation(createFunctionReqs, client.getUDFTable()); + assertUDFInformation(createFunctionReqs, client.getUDFTable(new TGetUdfTableReq(Model.TREE))); TShowCQResp showCQResp = client.showCQ(); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), showCQResp.getStatus().getCode()); @@ -283,12 +288,16 @@ private List createUDF(SyncConfigNodeIServiceClient client) TCreateFunctionReq createFunctionReq1 = new TCreateFunctionReq("test1", "org.apache.iotdb.udf.UDTFExample", true) + .setModel(Model.TREE) + .setFunctionType(FunctionType.NONE) .setJarName(jarName) .setJarFile(jarFile) .setJarMD5(jarMD5); TCreateFunctionReq createFunctionReq2 = new TCreateFunctionReq("test2", "org.apache.iotdb.udf.UDTFExample", true) + .setModel(Model.TREE) + .setFunctionType(FunctionType.NONE) .setJarName(jarName) .setJarFile(jarFile) .setJarMD5(jarMD5); @@ -307,11 +316,13 @@ private List createUDF(SyncConfigNodeIServiceClient client) } private void assertUDFInformation(List req, TGetUDFTableResp resp) { + Map nameToReqMap = + req.stream().collect(Collectors.toMap(r -> r.getUdfName().toUpperCase(), r -> r)); for (int i = 0; i < req.size(); i++) { - TCreateFunctionReq createFunctionReq = req.get(i); UDFInformation udfInformation = UDFInformation.deserialize(resp.getAllUDFInformation().get(i)); - + assertTrue(nameToReqMap.containsKey(udfInformation.getFunctionName())); + TCreateFunctionReq createFunctionReq = nameToReqMap.get(udfInformation.getFunctionName()); assertEquals(createFunctionReq.getUdfName().toUpperCase(), udfInformation.getFunctionName()); assertEquals(createFunctionReq.getClassName(), udfInformation.getClassName()); assertEquals(createFunctionReq.getJarName(), udfInformation.getJarName()); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java index d31e221724950..2ca034497f095 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java @@ -162,12 +162,10 @@ public void showClusterAndNodesTest() throws Exception { clusterParameters.getSchemaReplicationFactor()); Assert.assertEquals( expectedParameters.getDataRegionPerDataNode(), - clusterParameters.getDataRegionPerDataNode(), - 0.01); + clusterParameters.getDataRegionPerDataNode()); Assert.assertEquals( expectedParameters.getSchemaRegionPerDataNode(), - clusterParameters.getSchemaRegionPerDataNode(), - 0.01); + clusterParameters.getSchemaRegionPerDataNode()); Assert.assertEquals( expectedParameters.getDiskSpaceWarningThreshold(), clusterParameters.getDiskSpaceWarningThreshold(), @@ -241,7 +239,7 @@ public void removeAndStopConfigNodeTest() throws Exception { } // Test stop ConfigNode - status = client.stopConfigNode(removedConfigNodeLocation); + status = client.stopAndClearConfigNode(removedConfigNodeLocation); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java index cf29f649c3cf0..7a81810a1f533 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java @@ -56,8 +56,8 @@ public class IoTDBDatabaseSetAndDeleteIT { @Before public void setUp() throws Exception { - // Init 1C0D environment - EnvFactory.getEnv().initClusterEnvironment(1, 0); + // Init 1C1D environment + EnvFactory.getEnv().initClusterEnvironment(1, 1); } @After diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBAutoRegionGroupExtension2IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBAutoRegionGroupExtension2IT.java new file mode 100644 index 0000000000000..5d00ef023ebc5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBAutoRegionGroupExtension2IT.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBAutoRegionGroupExtension2IT { + + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBAutoRegionGroupExtension2IT.class); + + private static final String testDataRegionGroupExtensionPolicy = "AUTO"; + private static final String testConsensusProtocolClass = ConsensusFactory.IOT_CONSENSUS; + private static final int testReplicationFactor = 3; + + private static final String database = "root.db"; + private static final long testTimePartitionInterval = 604800000; + private static final int testMinDataRegionGroupNum = 3; + private static final int testDataNodeNum = 3; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(testConsensusProtocolClass) + .setDataReplicationFactor(testReplicationFactor) + .setDataRegionGroupExtensionPolicy(testDataRegionGroupExtensionPolicy) + .setTimePartitionInterval(testTimePartitionInterval); + // Init 1C3D environment + EnvFactory.getEnv().initClusterEnvironment(1, testDataNodeNum); + } + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAutoRegionGroupExtensionPolicy2() + throws ClientManagerException, IOException, InterruptedException, TException { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + TSStatus status = + client.setDatabase( + new TDatabaseSchema(database).setMinDataRegionGroupNum(testMinDataRegionGroupNum)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + // Shutdown 1 DataNode + EnvFactory.getEnv().shutdownDataNode(1); + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(1)), + Collections.singletonList(NodeStatus.Unknown)); + + // Create 3 DataPartitions to extend 3 DataRegionGroups + for (int i = 0; i < testMinDataRegionGroupNum; i++) { + Map> partitionSlotsMap = + ConfigNodeTestUtils.constructPartitionSlotsMap( + database, i, i + 1, i, i + 1, testTimePartitionInterval); + TDataPartitionReq dataPartitionReq = new TDataPartitionReq(partitionSlotsMap); + TDataPartitionTableResp dataPartitionTableResp = null; + for (int retry = 0; retry < 5; retry++) { + // Build new Client since it's unstable in Win8 environment + try (SyncConfigNodeIServiceClient configNodeClient = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + dataPartitionTableResp = + configNodeClient.getOrCreateDataPartitionTable(dataPartitionReq); + if (dataPartitionTableResp != null) { + break; + } + } catch (Exception e) { + // Retry sometimes in order to avoid request timeout + LOGGER.error(e.getMessage()); + TimeUnit.SECONDS.sleep(1); + } + } + Assert.assertNotNull(dataPartitionTableResp); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + dataPartitionTableResp.getStatus().getCode()); + Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable()); + ConfigNodeTestUtils.checkDataPartitionTable( + database, + i, + i + 1, + i, + i + 1, + testTimePartitionInterval, + dataPartitionTableResp.getDataPartitionTable()); + } + + // Restart DataNode + EnvFactory.getEnv().startDataNode(1); + + // Check DataRegionGroups + TShowRegionResp resp = client.showRegion(new TShowRegionReq()); + Map counter = new HashMap<>(); + resp.getRegionInfoList() + .forEach( + regionInfo -> + counter + .computeIfAbsent(regionInfo.getDataNodeId(), empty -> new AtomicInteger(0)) + .incrementAndGet()); + counter.forEach((dataNodeId, regionCount) -> Assert.assertEquals(3, regionCount.get())); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBAutoRegionGroupExtensionIT2.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBAutoRegionGroupExtensionIT2.java deleted file mode 100644 index f091bafd542cb..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBAutoRegionGroupExtensionIT2.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.partition; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; -import org.apache.iotdb.commons.client.exception.ClientManagerException; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils; -import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; -import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; -import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; -import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.thrift.TException; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@RunWith(IoTDBTestRunner.class) -@Category({ClusterIT.class}) -public class IoTDBAutoRegionGroupExtensionIT2 { - - private static final Logger LOGGER = - LoggerFactory.getLogger(IoTDBAutoRegionGroupExtensionIT2.class); - - private static final String testDataRegionGroupExtensionPolicy = "AUTO"; - private static final String testConsensusProtocolClass = ConsensusFactory.IOT_CONSENSUS; - private static final int testReplicationFactor = 3; - - private static final String database = "root.db"; - private static final long testTimePartitionInterval = 604800000; - private static final int testMinDataRegionGroupNum = 3; - private static final int testDataNodeNum = 3; - - @Before - public void setUp() throws Exception { - EnvFactory.getEnv() - .getConfig() - .getCommonConfig() - .setDataRegionConsensusProtocolClass(testConsensusProtocolClass) - .setDataReplicationFactor(testReplicationFactor) - .setDataRegionGroupExtensionPolicy(testDataRegionGroupExtensionPolicy) - .setTimePartitionInterval(testTimePartitionInterval); - // Init 1C3D environment - EnvFactory.getEnv().initClusterEnvironment(1, testDataNodeNum); - } - - @After - public void tearDown() { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void testAutoRegionGroupExtensionPolicy2() - throws ClientManagerException, IOException, InterruptedException, TException { - try (SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - TSStatus status = - client.setDatabase( - new TDatabaseSchema(database).setMinDataRegionGroupNum(testMinDataRegionGroupNum)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - // Shutdown 1 DataNode - EnvFactory.getEnv().shutdownDataNode(1); - EnvFactory.getEnv() - .ensureNodeStatus( - Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(1)), - Collections.singletonList(NodeStatus.Unknown)); - - // Create 3 DataPartitions to extend 3 DataRegionGroups - for (int i = 0; i < testMinDataRegionGroupNum; i++) { - Map> partitionSlotsMap = - ConfigNodeTestUtils.constructPartitionSlotsMap( - database, i, i + 1, i, i + 1, testTimePartitionInterval); - TDataPartitionReq dataPartitionReq = new TDataPartitionReq(partitionSlotsMap); - TDataPartitionTableResp dataPartitionTableResp = null; - for (int retry = 0; retry < 5; retry++) { - // Build new Client since it's unstable in Win8 environment - try (SyncConfigNodeIServiceClient configNodeClient = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - dataPartitionTableResp = - configNodeClient.getOrCreateDataPartitionTable(dataPartitionReq); - if (dataPartitionTableResp != null) { - break; - } - } catch (Exception e) { - // Retry sometimes in order to avoid request timeout - LOGGER.error(e.getMessage()); - TimeUnit.SECONDS.sleep(1); - } - } - Assert.assertNotNull(dataPartitionTableResp); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - dataPartitionTableResp.getStatus().getCode()); - Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable()); - ConfigNodeTestUtils.checkDataPartitionTable( - database, - i, - i + 1, - i, - i + 1, - testTimePartitionInterval, - dataPartitionTableResp.getDataPartitionTable()); - } - - // Restart DataNode - EnvFactory.getEnv().startDataNode(1); - - // Check DataRegionGroups - TShowRegionResp resp = client.showRegion(new TShowRegionReq()); - Map counter = new HashMap<>(); - resp.getRegionInfoList() - .forEach( - regionInfo -> - counter - .computeIfAbsent(regionInfo.getDataNodeId(), empty -> new AtomicInteger(0)) - .incrementAndGet()); - counter.forEach((dataNodeId, regionCount) -> Assert.assertEquals(3, regionCount.get())); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java index ed2423e304594..2a0ccffe60cd0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java @@ -388,27 +388,48 @@ public void testPartitionAllocation() throws Exception { testTimePartitionInterval, dataPartitionTableResp.getDataPartitionTable()); - // Check Region count and status int runningCnt = 0; int unknownCnt = 0; int readOnlyCnt = 0; int removingCnt = 0; - TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { - if (RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) { - runningCnt += 1; - } else if (RegionStatus.Unknown.getStatus().equals(regionInfo.getStatus())) { - unknownCnt += 1; - } else if (RegionStatus.Removing.getStatus().equals(regionInfo.getStatus())) { - removingCnt += 1; - } else if (RegionStatus.ReadOnly.getStatus().equals(regionInfo.getStatus())) { - readOnlyCnt += 1; + TShowRegionResp showRegionResp; + + // Check Region count and status + for (int retry = 0; retry < 30; retry++) { + runningCnt = 0; + unknownCnt = 0; + readOnlyCnt = 0; + removingCnt = 0; + showRegionResp = client.showRegion(new TShowRegionReq()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); + for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { + if (RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) { + runningCnt += 1; + } else if (RegionStatus.Unknown.getStatus().equals(regionInfo.getStatus())) { + unknownCnt += 1; + } else if (RegionStatus.Removing.getStatus().equals(regionInfo.getStatus())) { + removingCnt += 1; + } else if (RegionStatus.ReadOnly.getStatus().equals(regionInfo.getStatus())) { + readOnlyCnt += 1; + } + } + + if (runningCnt == 9 && removingCnt == 0 && readOnlyCnt == 1 && unknownCnt == 2) { + break; + } else { + LOGGER.info( + "Running: {}, Removing: {}, ReadOnly:{}, Unknown:{}", + runningCnt, + removingCnt, + readOnlyCnt, + unknownCnt); + TimeUnit.SECONDS.sleep(1); } } - Assert.assertEquals(8, runningCnt); - Assert.assertEquals(1, removingCnt); + + Assert.assertEquals(9, runningCnt); + Assert.assertEquals(0, removingCnt); Assert.assertEquals(1, readOnlyCnt); Assert.assertEquals(2, unknownCnt); @@ -455,9 +476,15 @@ public void testPartitionAllocation() throws Exception { readOnlyCnt += 1; } } - if (runningCnt == 10 && unknownCnt == 0 && readOnlyCnt == 1 && removingCnt == 1) { + if (runningCnt == 11 && unknownCnt == 0 && readOnlyCnt == 1 && removingCnt == 0) { return; } + LOGGER.info( + "Running: {}, Removing: {}, ReadOnly:{}, Unknown:{}", + runningCnt, + removingCnt, + readOnlyCnt, + unknownCnt); TimeUnit.SECONDS.sleep(1); } Assert.fail("Region status is not correct after 30s of recovery"); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java index 9e94d0ddabb0e..93780ec68dc0b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.confignode.it.partition; -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; @@ -32,8 +31,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; -import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; -import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp; import org.apache.iotdb.confignode.rpc.thrift.TSetDataNodeStatusReq; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; @@ -122,133 +119,6 @@ public void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } - // TODO: Fix this when replica completion is supported - @Test - public void testRemovingDataNode() throws Exception { - try (SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - - /* Test getOrCreateSchemaPartition, ConfigNode should create SchemaPartition and return */ - TSchemaPartitionReq schemaPartitionReq = - new TSchemaPartitionReq() - .setPathPatternTree(ConfigNodeTestUtils.generatePatternTreeBuffer(new String[] {d0})); - TSchemaPartitionTableResp schemaPartitionTableResp = - client.getOrCreateSchemaPartitionTable(schemaPartitionReq); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - schemaPartitionTableResp.getStatus().getCode()); - Map> schemaPartitionTable = - schemaPartitionTableResp.getSchemaPartitionTable(); - // Successfully create a SchemaPartition - Assert.assertTrue(schemaPartitionTable.containsKey(sg)); - Assert.assertEquals(1, schemaPartitionTable.get(sg).size()); - - /* Check Region distribution */ - TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - // Create exactly one RegionGroup - Assert.assertEquals(3, showRegionResp.getRegionInfoListSize()); - // Each DataNode has exactly one Region - Set dataNodeIdSet = new HashSet<>(); - showRegionResp - .getRegionInfoList() - .forEach(regionInfo -> dataNodeIdSet.add(regionInfo.getDataNodeId())); - Assert.assertEquals(3, dataNodeIdSet.size()); - - /* Change the NodeStatus of the test DataNode to Removing */ - TSetDataNodeStatusReq setDataNodeStatusReq = new TSetDataNodeStatusReq(); - DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId); - setDataNodeStatusReq.setTargetDataNode( - new TDataNodeLocation(defaultDataNode) - .setInternalEndPoint( - new TEndPoint() - .setIp(dataNodeWrapper.getInternalAddress()) - .setPort(dataNodeWrapper.getInternalPort()))); - setDataNodeStatusReq.setStatus(NodeStatus.Removing.getStatus()); - client.setDataNodeStatus(setDataNodeStatusReq); - // Waiting for heartbeat update - while (true) { - AtomicBoolean containRemoving = new AtomicBoolean(false); - TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - showDataNodesResp - .getDataNodesInfoList() - .forEach( - dataNodeInfo -> { - if (NodeStatus.Removing.getStatus().equals(dataNodeInfo.getStatus())) { - containRemoving.set(true); - } - }); - - if (containRemoving.get()) { - break; - } - TimeUnit.SECONDS.sleep(1); - } - - /* Test getOrCreateSchemaPartition, the result should be NO_ENOUGH_DATANODE */ - schemaPartitionReq = - new TSchemaPartitionReq() - .setPathPatternTree(ConfigNodeTestUtils.generatePatternTreeBuffer(new String[] {d1})); - schemaPartitionTableResp = client.getOrCreateSchemaPartitionTable(schemaPartitionReq); - Assert.assertEquals( - TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode(), - schemaPartitionTableResp.getStatus().getCode()); - - /* Register a new DataNode */ - EnvFactory.getEnv().registerNewDataNode(true); - - /* Test getOrCreateSchemaPartition, ConfigNode should create SchemaPartition and return */ - schemaPartitionReq = - new TSchemaPartitionReq() - .setPathPatternTree(ConfigNodeTestUtils.generatePatternTreeBuffer(new String[] {d1})); - schemaPartitionTableResp = client.getOrCreateSchemaPartitionTable(schemaPartitionReq); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - schemaPartitionTableResp.getStatus().getCode()); - schemaPartitionTable = schemaPartitionTableResp.getSchemaPartitionTable(); - // Successfully create a SchemaPartition - Assert.assertTrue(schemaPartitionTable.containsKey(sg)); - Assert.assertEquals(1, schemaPartitionTable.get(sg).size()); - - /* Check Region distribution */ - showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - // There should be 2 RegionGroups - Assert.assertEquals(6, showRegionResp.getRegionInfoListSize()); - // The new RegionGroup should keep away from the Removing DataNode - Map regionCounter = new ConcurrentHashMap<>(); - showRegionResp - .getRegionInfoList() - .forEach( - regionInfo -> - regionCounter - .computeIfAbsent(regionInfo.getDataNodeId(), empty -> new AtomicInteger(0)) - .getAndIncrement()); - dataNodeIdSet.forEach(dataNodeId -> regionCounter.get(dataNodeId).getAndDecrement()); - TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showDataNodesResp.getStatus().getCode()); - regionCounter.forEach( - (dataNodeId, regionCount) -> { - String nodeStatus = - showDataNodesResp.getDataNodesInfoList().stream() - .filter(dataNodeInfo -> dataNodeInfo.getDataNodeId() == dataNodeId) - .findFirst() - .orElse(new TDataNodeInfo().setStatus("ERROR")) - .getStatus(); - if (NodeStatus.Removing.getStatus().equals(nodeStatus)) { - Assert.assertEquals(0, regionCount.get()); - } else if (NodeStatus.Running.getStatus().equals(nodeStatus)) { - Assert.assertEquals(1, regionCount.get()); - } else { - Assert.fail(); - } - }); - } - } - // TODO: Fix this when replica completion is supported @Test public void testReadOnlyDataNode() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionGetterIT.java index 91a50cd20f785..65ff81cb8b1ca 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionGetterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionGetterIT.java @@ -52,6 +52,8 @@ import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -61,6 +63,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.HashSet; @@ -345,12 +348,12 @@ public void testGetSlots() throws Exception { final String sg0 = "root.sg0"; final String sg1 = "root.sg1"; - final String d00 = sg0 + ".d0.s"; - final String d01 = sg0 + ".d1.s"; - final String d10 = sg1 + ".d0.s"; - final String d11 = sg1 + ".d1.s"; + final IDeviceID d00 = Factory.DEFAULT_FACTORY.create(sg0 + ".d0.s"); + final IDeviceID d01 = Factory.DEFAULT_FACTORY.create(sg0 + ".d1.s"); + final IDeviceID d10 = Factory.DEFAULT_FACTORY.create(sg1 + ".d0.s"); + final IDeviceID d11 = Factory.DEFAULT_FACTORY.create(sg1 + ".d1.s"); - String[] devices = new String[] {d00, d01, d10, d11}; + IDeviceID[] devices = new IDeviceID[] {d00, d01, d10, d11}; try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { @@ -430,8 +433,10 @@ public void testGetSlots() throws Exception { // Get RegionId with device TGetRegionIdReq deviceReq = new TGetRegionIdReq(TConsensusGroupType.DataRegion); - for (String device : devices) { - deviceReq.setDevice(device); + for (IDeviceID device : devices) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + device.serialize(baos); + deviceReq.setDevice(baos.toByteArray()); TGetRegionIdResp resp = client.getRegionId(deviceReq); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java new file mode 100644 index 0000000000000..3f3596746f005 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBPartitionTableAutoCleanIT { + + private static final String TREE_DATABASE_PREFIX = "root.db.g_"; + private static final String TABLE_DATABASE_PREFIX = "database_"; + + private static final int TEST_REPLICATION_FACTOR = 1; + private static final long TEST_TIME_PARTITION_INTERVAL = 604800000; + private static final long TEST_TTL_CHECK_INTERVAL = 5_000; + + private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT = + new TTimePartitionSlot() + .setStartTime( + System.currentTimeMillis() + / TEST_TIME_PARTITION_INTERVAL + * TEST_TIME_PARTITION_INTERVAL); + private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaReplicationFactor(TEST_REPLICATION_FACTOR) + .setDataReplicationFactor(TEST_REPLICATION_FACTOR) + .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL) + .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL); + + // Init 1C1D environment + EnvFactory.getEnv().initClusterEnvironment(1, 1); + } + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAutoCleanPartitionTableForTreeModel() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + // Create databases and insert test data + for (int i = 0; i < 3; i++) { + String databaseName = String.format("%s%d", TREE_DATABASE_PREFIX, i); + statement.execute(String.format("CREATE DATABASE %s", databaseName)); + statement.execute( + String.format( + "CREATE TIMESERIES %s.s WITH DATATYPE=INT64,ENCODING=PLAIN", databaseName)); + // Insert expired data + statement.execute( + String.format( + "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); + // Insert existed data + statement.execute( + String.format( + "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); + } + // Let db0.TTL > device.TTL, the valid TTL should be the bigger one + statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL)); + statement.execute(String.format("SET TTL TO %s0.s %d", TREE_DATABASE_PREFIX, 10)); + // Let db1.TTL < device.TTL, the valid TTL should be the bigger one + statement.execute(String.format("SET TTL TO %s1 %d", TREE_DATABASE_PREFIX, 10)); + statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL)); + // Set TTL to path db2.** + statement.execute(String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL)); + } + + TDataPartitionReq req = new TDataPartitionReq(); + for (int i = 0; i < 3; i++) { + req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>()); + } + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (int retry = 0; retry < 120; retry++) { + boolean partitionTableAutoCleaned = true; + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { + partitionTableAutoCleaned = + resp.getDataPartitionTable().entrySet().stream() + .flatMap(e1 -> e1.getValue().entrySet().stream()) + .allMatch(e2 -> e2.getValue().size() == 1); + } + if (partitionTableAutoCleaned) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + } + Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } + + @Test + public void testAutoCleanPartitionTableForTableModel() throws Exception { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Create databases and insert test data + String databaseName = TABLE_DATABASE_PREFIX; + statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", databaseName)); + statement.execute(String.format("USE %s", databaseName)); + statement.execute("CREATE TABLE tb (time TIMESTAMP TIME, s int64 FIELD)"); + // Insert expired data + statement.execute( + String.format( + "INSERT INTO tb(time, s) VALUES (%d, %d)", + TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); + // Insert existed data + statement.execute( + String.format( + "INSERT INTO tb(time, s) VALUES (%d, %d)", TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); + statement.execute(String.format("USE %s", TABLE_DATABASE_PREFIX)); + statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", TEST_TTL)); + } + + TDataPartitionReq req = new TDataPartitionReq(); + req.putToPartitionSlotsMap(TABLE_DATABASE_PREFIX, new TreeMap<>()); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (int retry = 0; retry < 120; retry++) { + boolean partitionTableAutoCleaned = true; + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { + partitionTableAutoCleaned = + resp.getDataPartitionTable().entrySet().stream() + .flatMap(e1 -> e1.getValue().entrySet().stream()) + .allMatch(e2 -> e2.getValue().size() == 1); + } + if (partitionTableAutoCleaned) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + } + Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java new file mode 100644 index 0000000000000..69ab3a03d2832 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBPartitionTableAutoCleanUSIT { + + private static final String TREE_DATABASE_PREFIX = "root.db.g_"; + private static final String TABLE_DATABASE_PREFIX = "database_"; + + private static final int TEST_REPLICATION_FACTOR = 1; + private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000; + private static final long TEST_TTL_CHECK_INTERVAL = 5_000; + + private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT = + new TTimePartitionSlot() + .setStartTime( + System.currentTimeMillis() + * 1000L + / TEST_TIME_PARTITION_INTERVAL_IN_MS + * TEST_TIME_PARTITION_INTERVAL_IN_MS); + private static final long TEST_TTL_IN_MS = 7 * TEST_TIME_PARTITION_INTERVAL_IN_MS; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaReplicationFactor(TEST_REPLICATION_FACTOR) + .setDataReplicationFactor(TEST_REPLICATION_FACTOR) + .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL_IN_MS) + .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL) + // Note that the time precision of IoTDB is us in this IT + .setTimestampPrecision("us"); + + // Init 1C1D environment + EnvFactory.getEnv().initClusterEnvironment(1, 1); + } + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAutoCleanPartitionTableForTreeModel() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + // Create databases and insert test data + for (int i = 0; i < 3; i++) { + String databaseName = String.format("%s%d", TREE_DATABASE_PREFIX, i); + statement.execute(String.format("CREATE DATABASE %s", databaseName)); + statement.execute( + String.format( + "CREATE TIMESERIES %s.s WITH DATATYPE=INT64,ENCODING=PLAIN", databaseName)); + // Insert expired data + statement.execute( + String.format( + "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000, -1)); + // Insert existed data + statement.execute( + String.format( + "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); + } + // Let db0.TTL > device.TTL, the valid TTL should be the bigger one + statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS)); + statement.execute(String.format("SET TTL TO %s0.s %d", TREE_DATABASE_PREFIX, 10)); + // Let db1.TTL < device.TTL, the valid TTL should be the bigger one + statement.execute(String.format("SET TTL TO %s1 %d", TREE_DATABASE_PREFIX, 10)); + statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS)); + // Set TTL to path db2.** + statement.execute( + String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS)); + } + TDataPartitionReq req = new TDataPartitionReq(); + for (int i = 0; i < 3; i++) { + req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>()); + } + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (int retry = 0; retry < 120; retry++) { + boolean partitionTableAutoCleaned = true; + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { + partitionTableAutoCleaned = + resp.getDataPartitionTable().entrySet().stream() + .flatMap(e1 -> e1.getValue().entrySet().stream()) + .allMatch(e2 -> e2.getValue().size() == 1); + } + if (partitionTableAutoCleaned) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + } + Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } + + @Test + public void testAutoCleanPartitionTableForTableModel() throws Exception { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Create databases and insert test data + String databaseName = TABLE_DATABASE_PREFIX; + statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", databaseName)); + statement.execute(String.format("USE %s", databaseName)); + statement.execute("CREATE TABLE tb (time TIMESTAMP TIME, s int64 FIELD)"); + // Insert expired data + statement.execute( + String.format( + "INSERT INTO tb(time, s) VALUES (%d, %d)", + TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000, -1)); + // Insert existed data + statement.execute( + String.format( + "INSERT INTO tb(time, s) VALUES (%d, %d)", TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); + statement.execute(String.format("USE %s", TABLE_DATABASE_PREFIX)); + statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", TEST_TTL_IN_MS)); + } + + TDataPartitionReq req = new TDataPartitionReq(); + req.putToPartitionSlotsMap(TABLE_DATABASE_PREFIX, new TreeMap<>()); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (int retry = 0; retry < 120; retry++) { + boolean partitionTableAutoCleaned = true; + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { + partitionTableAutoCleaned = + resp.getDataPartitionTable().entrySet().stream() + .flatMap(e1 -> e1.getValue().entrySet().stream()) + .allMatch(e2 -> e2.getValue().size() == 1); + } + if (partitionTableAutoCleaned) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + } + Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java index 771e7cdc5051c..469d6974cb583 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java @@ -22,11 +22,11 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.it.env.EnvFactory; diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFramework.java deleted file mode 100644 index 4a363d196e799..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFramework.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration; - -import org.apache.iotdb.commons.utils.KillPoint.KillNode; - -public class IoTDBRegionMigrateDataNodeCrashITFramework - extends IoTDBRegionMigrateReliabilityITFramework { - @SafeVarargs - public final > void success(T... dataNodeKillPoints) throws Exception { - successTest(1, 1, 1, 2, noKillPoints(), buildSet(dataNodeKillPoints), KillNode.ALL_NODES); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1.java new file mode 100644 index 0000000000000..7899d570bc4ed --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; + +import org.junit.Before; + +public class IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1 + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + } + + @SafeVarargs + public final > void success(T... dataNodeKillPoints) throws Exception { + successTest(1, 1, 1, 2, noKillPoints(), buildSet(dataNodeKillPoints), KillNode.ALL_NODES); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2.java new file mode 100644 index 0000000000000..e48178600739b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; + +public class IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 + extends IoTDBRegionOperationReliabilityITFramework { + + @SafeVarargs + public final > void success(T... dataNodeKillPoints) throws Exception { + successTest(1, 1, 1, 2, noKillPoints(), buildSet(dataNodeKillPoints), KillNode.ALL_NODES); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java deleted file mode 100644 index d52251b4a1a56..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java +++ /dev/null @@ -1,752 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration; - -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; -import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.commons.utils.KillPoint.KillNode; -import org.apache.iotdb.commons.utils.KillPoint.KillPoint; -import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.env.cluster.env.AbstractEnv; -import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; -import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.itbase.exception.InconsistentDataException; -import org.apache.iotdb.metrics.utils.SystemType; - -import org.apache.thrift.TException; -import org.awaitility.Awaitility; -import org.awaitility.core.ConditionTimeoutException; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Proxy; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentHashMap.KeySetView; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -public class IoTDBRegionMigrateReliabilityITFramework { - private static final Logger LOGGER = - LoggerFactory.getLogger(IoTDBRegionMigrateReliabilityITFramework.class); - private static final String INSERTION1 = - "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(100, 1, 2)"; - private static final String INSERTION2 = - "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(101, 3, 4)"; - private static final String FLUSH_COMMAND = "flush"; - private static final String SHOW_REGIONS = "show regions"; - private static final String SHOW_DATANODES = "show datanodes"; - private static final String COUNT_TIMESERIES = "select count(*) from root.sg.**"; - private static final String REGION_MIGRATE_COMMAND_FORMAT = "migrate region %d from %d to %d"; - private static final String CONFIGURATION_FILE_NAME = "configuration.dat"; - ExecutorService executorService = IoTDBThreadPoolFactory.newCachedThreadPool("regionMigrateIT"); - - public static Consumer actionOfKillNode = - context -> { - context.getNodeWrapper().stopForcibly(); - LOGGER.info("Node {} stopped.", context.getNodeWrapper().getId()); - Assert.assertFalse(context.getNodeWrapper().isAlive()); - if (context.getNodeWrapper() instanceof ConfigNodeWrapper) { - context.getNodeWrapper().start(); - LOGGER.info("Node {} restarted.", context.getNodeWrapper().getId()); - Assert.assertTrue(context.getNodeWrapper().isAlive()); - } - }; - - public static Consumer actionOfRestartCluster = - context -> { - context.getEnv().getNodeWrapperList().parallelStream() - .forEach(AbstractNodeWrapper::stopForcibly); - LOGGER.info("Cluster has been stopped"); - context.getEnv().getNodeWrapperList().parallelStream().forEach(AbstractNodeWrapper::start); - LOGGER.info("Cluster has been restarted"); - }; - - @Before - public void setUp() throws Exception { - EnvFactory.getEnv() - .getConfig() - .getCommonConfig() - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - } - - @After - public void tearDown() throws InterruptedException { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - public void successTest( - final int dataReplicateFactor, - final int schemaReplicationFactor, - final int configNodeNum, - final int dataNodeNum, - KeySetView killConfigNodeKeywords, - KeySetView killDataNodeKeywords, - KillNode killNode) - throws Exception { - generalTestWithAllOptions( - dataReplicateFactor, - schemaReplicationFactor, - configNodeNum, - dataNodeNum, - killConfigNodeKeywords, - killDataNodeKeywords, - actionOfKillNode, - true, - killNode); - } - - public void failTest( - final int dataReplicateFactor, - final int schemaReplicationFactor, - final int configNodeNum, - final int dataNodeNum, - KeySetView killConfigNodeKeywords, - KeySetView killDataNodeKeywords, - KillNode killNode) - throws Exception { - generalTestWithAllOptions( - dataReplicateFactor, - schemaReplicationFactor, - configNodeNum, - dataNodeNum, - killConfigNodeKeywords, - killDataNodeKeywords, - actionOfKillNode, - false, - killNode); - } - - public void killClusterTest( - KeySetView configNodeKeywords, boolean expectMigrateSuccess) - throws Exception { - generalTestWithAllOptions( - 2, - 3, - 3, - 3, - configNodeKeywords, - noKillPoints(), - actionOfRestartCluster, - expectMigrateSuccess, - KillNode.ALL_NODES); - } - - // region general test - - public void generalTestWithAllOptions( - final int dataReplicateFactor, - final int schemaReplicationFactor, - final int configNodeNum, - final int dataNodeNum, - KeySetView configNodeKeywords, - KeySetView dataNodeKeywords, - Consumer actionWhenDetectKeyWords, - final boolean expectMigrateSuccess, - KillNode killNode) - throws Exception { - // prepare env - EnvFactory.getEnv() - .getConfig() - .getCommonConfig() - .setDataReplicationFactor(dataReplicateFactor) - .setSchemaReplicationFactor(schemaReplicationFactor); - EnvFactory.getEnv().registerConfigNodeKillPoints(new ArrayList<>(configNodeKeywords)); - EnvFactory.getEnv().registerDataNodeKillPoints(new ArrayList<>(dataNodeKeywords)); - EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum); - - try (final Connection connection = closeQuietly(EnvFactory.getEnv().getConnection()); - final Statement statement = closeQuietly(connection.createStatement()); - SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - statement.execute(INSERTION1); - - ResultSet result = statement.executeQuery(SHOW_REGIONS); - Map> regionMap = getRegionMap(result); - - result = statement.executeQuery(SHOW_DATANODES); - Set allDataNodeId = new HashSet<>(); - while (result.next()) { - allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); - } - - final int selectedRegion = selectRegion(regionMap); - final int originalDataNode = selectOriginalDataNode(regionMap, selectedRegion); - final int destDataNode = selectDestDataNode(allDataNodeId, regionMap, selectedRegion); - - checkRegionFileExist(originalDataNode); - checkPeersExist(regionMap.get(selectedRegion), originalDataNode, selectedRegion); - - try { - awaitUntilFlush(statement, originalDataNode); - } catch (ConditionTimeoutException e) { - LOGGER.error("Flush timeout:", e); - Assert.fail(); - } - - // set kill points - if (killNode == KillNode.ORIGINAL_DATANODE) { - setDataNodeKillPoints( - Collections.singletonList( - EnvFactory.getEnv().dataNodeIdToWrapper(originalDataNode).get()), - dataNodeKeywords, - actionWhenDetectKeyWords); - } else if (killNode == KillNode.DESTINATION_DATANODE) { - setDataNodeKillPoints( - Collections.singletonList(EnvFactory.getEnv().dataNodeIdToWrapper(destDataNode).get()), - dataNodeKeywords, - actionWhenDetectKeyWords); - } else { - setConfigNodeKillPoints(configNodeKeywords, actionWhenDetectKeyWords); - setDataNodeKillPoints( - EnvFactory.getEnv().getDataNodeWrapperList(), - dataNodeKeywords, - actionWhenDetectKeyWords); - } - - LOGGER.info("DataNode set before migration: {}", regionMap.get(selectedRegion)); - - System.out.println( - "originalDataNode: " - + EnvFactory.getEnv().dataNodeIdToWrapper(originalDataNode).get().getNodePath()); - System.out.println( - "destDataNode: " - + EnvFactory.getEnv().dataNodeIdToWrapper(destDataNode).get().getNodePath()); - - // region migration start - statement.execute(buildRegionMigrateCommand(selectedRegion, originalDataNode, destDataNode)); - - boolean success = false; - try { - awaitUntilSuccess(client, selectedRegion, originalDataNode, destDataNode); - success = true; - } catch (ConditionTimeoutException e) { - if (expectMigrateSuccess) { - LOGGER.error("Region migrate failed", e); - Assert.fail(); - } - } - if (!expectMigrateSuccess && success) { - LOGGER.error("Region migrate succeeded unexpectedly"); - Assert.fail(); - } - - // make sure all kill points have been triggered - checkKillPointsAllTriggered(configNodeKeywords); - checkKillPointsAllTriggered(dataNodeKeywords); - - // check the remaining file - if (success) { - checkRegionFileClearIfNodeAlive(originalDataNode); - checkRegionFileExistIfNodeAlive(destDataNode); - checkPeersClearIfNodeAlive(allDataNodeId, originalDataNode, selectedRegion); - checkClusterStillWritable(); - } else { - checkRegionFileClearIfNodeAlive(destDataNode); - checkRegionFileExistIfNodeAlive(originalDataNode); - checkPeersClearIfNodeAlive(allDataNodeId, destDataNode, selectedRegion); - } - - LOGGER.info("test pass"); - } catch (InconsistentDataException ignore) { - - } - } - - private void restartDataNodes(List dataNodeWrappers) { - dataNodeWrappers.parallelStream() - .forEach( - nodeWrapper -> { - nodeWrapper.stop(); - Awaitility.await() - .atMost(1, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .until(() -> !nodeWrapper.isAlive()); - LOGGER.info("Node {} stopped.", nodeWrapper.getId()); - nodeWrapper.start(); - Awaitility.await() - .atMost(1, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .until(nodeWrapper::isAlive); - try { - TimeUnit.SECONDS.sleep(10); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - LOGGER.info("Node {} restarted.", nodeWrapper.getId()); - }); - } - - private void setConfigNodeKillPoints( - KeySetView killConfigNodeKeywords, Consumer action) { - EnvFactory.getEnv() - .getConfigNodeWrapperList() - .forEach( - configNodeWrapper -> - executorService.submit( - () -> - doActionWhenDetectKeywords( - configNodeWrapper, killConfigNodeKeywords, action))); - } - - private void setDataNodeKillPoints( - List dataNodeWrappers, - KeySetView killDataNodeKeywords, - Consumer action) { - dataNodeWrappers.forEach( - dataNodeWrapper -> - executorService.submit( - () -> doActionWhenDetectKeywords(dataNodeWrapper, killDataNodeKeywords, action))); - } - - /** - * Monitor the node's log and kill it when detect specific log. - * - * @param nodeWrapper Easy to understand - * @param keywords When detect these keywords in node's log, stop the node forcibly - */ - private static void doActionWhenDetectKeywords( - AbstractNodeWrapper nodeWrapper, - KeySetView keywords, - Consumer action) { - if (keywords.isEmpty()) { - return; - } - final String logFileName; - if (nodeWrapper instanceof ConfigNodeWrapper) { - logFileName = "log_confignode_all.log"; - } else { - logFileName = "log_datanode_all.log"; - } - SystemType type = SystemType.getSystemType(); - ProcessBuilder builder; - if (type == SystemType.LINUX || type == SystemType.MAC) { - builder = - new ProcessBuilder( - "tail", - "-f", - nodeWrapper.getNodePath() + File.separator + "logs" + File.separator + logFileName); - } else if (type == SystemType.WINDOWS) { - builder = - new ProcessBuilder( - "powershell", - "-Command", - "Get-Content " - + nodeWrapper.getNodePath() - + File.separator - + "logs" - + File.separator - + logFileName - + " -Wait"); - } else { - throw new UnsupportedOperationException("Unsupported system type " + type); - } - builder.redirectErrorStream(true); - - try { - Process process = builder.start(); - try (BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line; - while ((line = reader.readLine()) != null) { - // if trigger more than one keyword at a same time, test code may have mistakes - Assert.assertTrue( - line, - keywords.stream().map(KillPoint::addKillPointPrefix).filter(line::contains).count() - <= 1); - String finalLine = line; - Optional detectedKeyword = - keywords.stream() - .filter(keyword -> finalLine.contains(KillPoint.addKillPointPrefix(keyword))) - .findAny(); - if (detectedKeyword.isPresent()) { - // each keyword only trigger once - keywords.remove(detectedKeyword.get()); - action.accept(new KillPointContext(nodeWrapper, (AbstractEnv) EnvFactory.getEnv())); - LOGGER.info("Kill point triggered: {}", detectedKeyword.get()); - } - if (keywords.isEmpty()) { - break; - } - } - } catch (AssertionError e) { - LOGGER.error("gg", e); - throw e; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - void checkKillPointsAllTriggered(KeySetView killPoints) { - if (!killPoints.isEmpty()) { - killPoints.forEach(killPoint -> LOGGER.error("Kill point {} not triggered", killPoint)); - Assert.fail("Some kill points was not triggered"); - } - } - - private static String buildRegionMigrateCommand(int who, int from, int to) { - String result = String.format(REGION_MIGRATE_COMMAND_FORMAT, who, from, to); - LOGGER.info(result); - return result; - } - - private static Map> getRegionMap(ResultSet showRegionsResult) - throws SQLException { - Map> regionMap = new HashMap<>(); - while (showRegionsResult.next()) { - if (String.valueOf(TConsensusGroupType.DataRegion) - .equals(showRegionsResult.getString(ColumnHeaderConstant.TYPE))) { - int regionId = showRegionsResult.getInt(ColumnHeaderConstant.REGION_ID); - int dataNodeId = showRegionsResult.getInt(ColumnHeaderConstant.DATA_NODE_ID); - regionMap.computeIfAbsent(regionId, id -> new HashSet<>()).add(dataNodeId); - } - } - return regionMap; - } - - private static Map> getRegionMap(List regionInfoList) { - Map> regionMap = new HashMap<>(); - regionInfoList.forEach( - regionInfo -> { - int regionId = regionInfo.getConsensusGroupId().getId(); - regionMap - .computeIfAbsent(regionId, regionId1 -> new HashSet<>()) - .add(regionInfo.getDataNodeId()); - }); - return regionMap; - } - - private static int selectRegion(Map> regionMap) { - return regionMap.keySet().stream().findAny().orElseThrow(() -> new RuntimeException("gg")); - } - - private static int selectOriginalDataNode( - Map> regionMap, int selectedRegion) { - return regionMap.get(selectedRegion).stream() - .findAny() - .orElseThrow(() -> new RuntimeException("cannot find original DataNode")); - } - - private static int selectDestDataNode( - Set dataNodeSet, Map> regionMap, int selectedRegion) { - return dataNodeSet.stream() - .filter(dataNodeId -> !regionMap.get(selectedRegion).contains(dataNodeId)) - .findAny() - .orElseThrow(() -> new RuntimeException("cannot find dest DataNode")); - } - - private static void awaitUntilFlush(Statement statement, int originalDataNode) { - long startTime = System.currentTimeMillis(); - File sequence = new File(buildDataPath(originalDataNode, true)); - File unsequence = new File(buildDataPath(originalDataNode, false)); - Awaitility.await() - .atMost(1, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .until( - () -> { - statement.execute(FLUSH_COMMAND); - int fileNum = 0; - if (sequence.exists() && sequence.listFiles() != null) { - fileNum += Objects.requireNonNull(sequence.listFiles()).length; - } - if (unsequence.exists() && unsequence.listFiles() != null) { - fileNum += Objects.requireNonNull(unsequence.listFiles()).length; - } - return fileNum > 0; - }); - LOGGER.info("DataNode {} has been flushed", originalDataNode); - LOGGER.info("Flush cost time: {}ms", System.currentTimeMillis() - startTime); - } - - private static void awaitUntilSuccess( - SyncConfigNodeIServiceClient client, - int selectedRegion, - int originalDataNode, - int destDataNode) { - AtomicReference> lastTimeDataNodes = new AtomicReference<>(); - AtomicReference lastException = new AtomicReference<>(); - AtomicReference clientRef = new AtomicReference<>(client); - try { - Awaitility.await() - .atMost(2, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .until( - () -> { - try { - TShowRegionResp resp = clientRef.get().showRegion(new TShowRegionReq()); - Map> newRegionMap = getRegionMap(resp.getRegionInfoList()); - Set dataNodes = newRegionMap.get(selectedRegion); - lastTimeDataNodes.set(dataNodes); - return !dataNodes.contains(originalDataNode) && dataNodes.contains(destDataNode); - } catch (TException e) { - clientRef.set( - (SyncConfigNodeIServiceClient) - EnvFactory.getEnv().getLeaderConfigNodeConnection()); - lastException.set(e); - return false; - } catch (Exception e) { - // Any exception can be ignored - lastException.set(e); - return false; - } - }); - } catch (ConditionTimeoutException e) { - if (lastTimeDataNodes.get() == null) { - LOGGER.error( - "maybe show regions fail, lastTimeDataNodes is null, last Exception:", - lastException.get()); - throw e; - } - String actualSetStr = lastTimeDataNodes.get().toString(); - lastTimeDataNodes.get().remove(originalDataNode); - lastTimeDataNodes.get().add(destDataNode); - String expectSetStr = lastTimeDataNodes.toString(); - LOGGER.error("DataNode Set {} is unexpected, expect {}", actualSetStr, expectSetStr); - if (lastException.get() == null) { - LOGGER.info("No exception during awaiting"); - } else { - LOGGER.error("Last exception during awaiting:", lastException.get()); - } - throw e; - } - LOGGER.info("DataNode set has been successfully changed to {}", lastTimeDataNodes.get()); - } - - private static void checkRegionFileExistIfNodeAlive(int dataNode) { - if (EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().isAlive()) { - checkRegionFileExist(dataNode); - } - } - - private static void checkRegionFileExist(int dataNode) { - File originalRegionDir = new File(buildRegionDirPath(dataNode)); - Assert.assertTrue(originalRegionDir.isDirectory()); - Assert.assertNotEquals(0, Objects.requireNonNull(originalRegionDir.listFiles()).length); - } - - private static void checkRegionFileClearIfNodeAlive(int dataNode) { - if (EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().isAlive()) { - checkRegionFileClear(dataNode); - } - } - - /** Check whether the original DataNode's region file has been deleted. */ - private static void checkRegionFileClear(int dataNode) { - File originalRegionDir = new File(buildRegionDirPath(dataNode)); - Assert.assertTrue(originalRegionDir.isDirectory()); - try { - Assert.assertEquals(0, Objects.requireNonNull(originalRegionDir.listFiles()).length); - } catch (AssertionError e) { - LOGGER.error( - "Original DataNode {} region file not clear, these files is still remain: {}", - dataNode, - Arrays.toString(originalRegionDir.listFiles())); - throw e; - } - LOGGER.info("Original DataNode {} region file clear", dataNode); - } - - private static void checkPeersExistIfNodeAlive( - Set dataNodes, int originalDataNode, int regionId) { - dataNodes.forEach( - targetDataNode -> checkPeerExistIfNodeAlive(targetDataNode, originalDataNode, regionId)); - } - - private static void checkPeersExist(Set dataNodes, int originalDataNode, int regionId) { - dataNodes.forEach(targetDataNode -> checkPeerExist(targetDataNode, originalDataNode, regionId)); - } - - private static void checkPeerExistIfNodeAlive( - int checkTargetDataNode, int originalDataNode, int regionId) { - if (EnvFactory.getEnv().dataNodeIdToWrapper(checkTargetDataNode).get().isAlive()) { - checkPeerExist(checkTargetDataNode, originalDataNode, regionId); - } - } - - private static void checkPeerExist(int checkTargetDataNode, int originalDataNode, int regionId) { - File expectExistedFile = - new File(buildConfigurationDataFilePath(checkTargetDataNode, originalDataNode, regionId)); - Assert.assertTrue( - "configuration file should exist, but it didn't: " + expectExistedFile.getPath(), - expectExistedFile.exists()); - } - - private static void checkPeersClearIfNodeAlive( - Set dataNodes, int originalDataNode, int regionId) { - dataNodes.stream() - .filter(dataNode -> dataNode != originalDataNode) - .forEach( - targetDataNode -> - checkPeerClearIfNodeAlive(targetDataNode, originalDataNode, regionId)); - } - - private static void checkPeersClear(Set dataNodes, int originalDataNode, int regionId) { - dataNodes.stream() - .filter(dataNode -> dataNode != originalDataNode) - .forEach(targetDataNode -> checkPeerClear(targetDataNode, originalDataNode, regionId)); - LOGGER.info("Peer clear"); - } - - private static void checkPeerClearIfNodeAlive( - int checkTargetDataNode, int originalDataNode, int regionId) { - if (EnvFactory.getEnv().dataNodeIdToWrapper(checkTargetDataNode).get().isAlive()) { - checkPeerClear(checkTargetDataNode, originalDataNode, regionId); - } - } - - private static void checkPeerClear(int checkTargetDataNode, int originalDataNode, int regionId) { - File expectDeletedFile = - new File(buildConfigurationDataFilePath(checkTargetDataNode, originalDataNode, regionId)); - Assert.assertFalse( - "configuration file should be deleted, but it didn't: " + expectDeletedFile.getPath(), - expectDeletedFile.exists()); - LOGGER.info("configuration file has been deleted: {}", expectDeletedFile.getPath()); - } - - private void checkClusterStillWritable() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - // check old data - ResultSet resultSet = statement.executeQuery(COUNT_TIMESERIES); - resultSet.next(); - Assert.assertEquals(1, resultSet.getLong(1)); - Assert.assertEquals(1, resultSet.getLong(2)); - LOGGER.info("Old data is still remain"); - // write new data - statement.execute(INSERTION2); - resultSet = statement.executeQuery(COUNT_TIMESERIES); - resultSet.next(); - Assert.assertEquals(2, resultSet.getLong(1)); - Assert.assertEquals(2, resultSet.getLong(2)); - LOGGER.info("Region group is still writable"); - } catch (SQLException e) { - LOGGER.error("Something wrong", e); - Assert.fail("Something wrong"); - } - } - - private static String buildRegionDirPath(int dataNode) { - String nodePath = EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().getNodePath(); - return nodePath - + File.separator - + IoTDBConstant.DATA_FOLDER_NAME - + File.separator - + "datanode" - + File.separator - + IoTDBConstant.CONSENSUS_FOLDER_NAME - + File.separator - + IoTDBConstant.DATA_REGION_FOLDER_NAME; - } - - private static String buildDataPath(int dataNode, boolean isSequence) { - String nodePath = EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().getNodePath(); - return nodePath - + File.separator - + IoTDBConstant.DATA_FOLDER_NAME - + File.separator - + "datanode" - + File.separator - + IoTDBConstant.DATA_FOLDER_NAME - + File.separator - + (isSequence ? IoTDBConstant.SEQUENCE_FOLDER_NAME : IoTDBConstant.UNSEQUENCE_FOLDER_NAME); - } - - private static String buildConfigurationDataFilePath( - int localDataNodeId, int remoteDataNodeId, int regionId) { - String configurationDatDirName = - buildRegionDirPath(localDataNodeId) + File.separator + "1_" + regionId; - String expectDeletedFileName = - IoTConsensusServerImpl.generateConfigurationDatFileName( - remoteDataNodeId, CONFIGURATION_FILE_NAME); - return configurationDatDirName + File.separator + expectDeletedFileName; - } - - protected static KeySetView noKillPoints() { - return ConcurrentHashMap.newKeySet(); - } - - @SafeVarargs - protected static > KeySetView buildSet(T... keywords) { - KeySetView result = ConcurrentHashMap.newKeySet(); - result.addAll( - Arrays.stream(keywords).map(KillPoint::enumToString).collect(Collectors.toList())); - return result; - } - - private static T closeQuietly(T t) { - InvocationHandler handler = - (proxy, method, args) -> { - try { - if (method.getName().equals("close")) { - try { - method.invoke(t, args); - } catch (Throwable e) { - LOGGER.warn("Exception happens during close(): ", e); - } - return null; - } else { - return method.invoke(t, args); - } - } catch (InvocationTargetException e) { - throw e.getTargetException(); - } - }; - return (T) - Proxy.newProxyInstance( - t.getClass().getClassLoader(), t.getClass().getInterfaces(), handler); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java new file mode 100644 index 0000000000000..5ef20ddd20992 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java @@ -0,0 +1,698 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.commons.utils.KillPoint.KillPoint; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; +import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.metrics.utils.SystemType; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.thrift.TException; +import org.apache.tsfile.read.common.Field; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentHashMap.KeySetView; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; + +public class IoTDBRegionOperationReliabilityITFramework { + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBRegionOperationReliabilityITFramework.class); + public static final String INSERTION1 = + "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(100, 1, 2)"; + private static final String INSERTION2 = + "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(101, 3, 4)"; + public static final String FLUSH_COMMAND = "flush on cluster"; + private static final String SHOW_REGIONS = "show regions"; + private static final String SHOW_DATANODES = "show datanodes"; + private static final String COUNT_TIMESERIES = "select count(*) from root.sg.**"; + private static final String REGION_MIGRATE_COMMAND_FORMAT = "migrate region %d from %d to %d"; + ExecutorService executorService = IoTDBThreadPoolFactory.newCachedThreadPool("regionMigrateIT"); + + public static Consumer actionOfKillNode = + context -> { + context.getNodeWrapper().stopForcibly(); + LOGGER.info("Node {} stopped.", context.getNodeWrapper().getId()); + Assert.assertFalse(context.getNodeWrapper().isAlive()); + if (context.getNodeWrapper() instanceof ConfigNodeWrapper) { + context.getNodeWrapper().start(); + LOGGER.info("Node {} restarted.", context.getNodeWrapper().getId()); + Assert.assertTrue(context.getNodeWrapper().isAlive()); + } + }; + + public static Consumer actionOfRestartCluster = + context -> { + context.getEnv().getNodeWrapperList().parallelStream() + .forEach(AbstractNodeWrapper::stopForcibly); + LOGGER.info("Cluster has been stopped"); + context.getEnv().getNodeWrapperList().parallelStream().forEach(AbstractNodeWrapper::start); + LOGGER.info("Cluster has been restarted"); + }; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS_V2) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + } + + @After + public void tearDown() throws InterruptedException { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + public void successTest( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + KeySetView killConfigNodeKeywords, + KeySetView killDataNodeKeywords, + KillNode killNode) + throws Exception { + generalTestWithAllOptions( + dataReplicateFactor, + schemaReplicationFactor, + configNodeNum, + dataNodeNum, + killConfigNodeKeywords, + killDataNodeKeywords, + actionOfKillNode, + true, + killNode); + } + + public void failTest( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + KeySetView killConfigNodeKeywords, + KeySetView killDataNodeKeywords, + KillNode killNode) + throws Exception { + generalTestWithAllOptions( + dataReplicateFactor, + schemaReplicationFactor, + configNodeNum, + dataNodeNum, + killConfigNodeKeywords, + killDataNodeKeywords, + actionOfKillNode, + false, + killNode); + } + + public void killClusterTest( + KeySetView configNodeKeywords, boolean expectMigrateSuccess) + throws Exception { + generalTestWithAllOptions( + 2, + 3, + 3, + 3, + configNodeKeywords, + noKillPoints(), + actionOfRestartCluster, + expectMigrateSuccess, + KillNode.ALL_NODES); + } + + // region general test + + public void generalTestWithAllOptions( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + KeySetView configNodeKeywords, + KeySetView dataNodeKeywords, + Consumer actionWhenDetectKeyWords, + final boolean expectMigrateSuccess, + KillNode killNode) + throws Exception { + // prepare env + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataReplicationFactor(dataReplicateFactor) + .setSchemaReplicationFactor(schemaReplicationFactor); + EnvFactory.getEnv().registerConfigNodeKillPoints(new ArrayList<>(configNodeKeywords)); + EnvFactory.getEnv().registerDataNodeKillPoints(new ArrayList<>(dataNodeKeywords)); + EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum); + + try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement()); + SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + // prepare data + statement.execute(INSERTION1); + statement.execute(FLUSH_COMMAND); + + // collect necessary information + Map> regionMap = getDataRegionMap(statement); + Set allDataNodeId = getAllDataNodes(statement); + + // select region migration related DataNodes + final int selectedRegion = selectRegion(regionMap); + final int originalDataNode = selectOriginalDataNode(regionMap, selectedRegion); + final int destDataNode = + selectDataNodeNotContainsRegion(allDataNodeId, regionMap, selectedRegion); + checkRegionFileExist(originalDataNode); + + // set kill points + if (killNode == KillNode.ORIGINAL_DATANODE) { + setDataNodeKillPoints( + Collections.singletonList( + EnvFactory.getEnv().dataNodeIdToWrapper(originalDataNode).get()), + dataNodeKeywords, + actionWhenDetectKeyWords); + } else if (killNode == KillNode.DESTINATION_DATANODE) { + setDataNodeKillPoints( + Collections.singletonList(EnvFactory.getEnv().dataNodeIdToWrapper(destDataNode).get()), + dataNodeKeywords, + actionWhenDetectKeyWords); + } else { + setConfigNodeKillPoints(configNodeKeywords, actionWhenDetectKeyWords); + setDataNodeKillPoints( + EnvFactory.getEnv().getDataNodeWrapperList(), + dataNodeKeywords, + actionWhenDetectKeyWords); + } + + LOGGER.info("DataNode set before migration: {}", regionMap.get(selectedRegion)); + + System.out.println( + "originalDataNode: " + + EnvFactory.getEnv().dataNodeIdToWrapper(originalDataNode).get().getNodePath()); + System.out.println( + "destDataNode: " + + EnvFactory.getEnv().dataNodeIdToWrapper(destDataNode).get().getNodePath()); + + // region migration start + statement.execute(buildRegionMigrateCommand(selectedRegion, originalDataNode, destDataNode)); + + boolean success = false; + Predicate migrateRegionPredicate = + tShowRegionResp -> { + Map> newRegionMap = + getRegionMap(tShowRegionResp.getRegionInfoList()); + Set dataNodes = newRegionMap.get(selectedRegion); + return !dataNodes.contains(originalDataNode) && dataNodes.contains(destDataNode); + }; + try { + awaitUntilSuccess( + client, + migrateRegionPredicate, + Optional.of(destDataNode), + Optional.of(originalDataNode)); + success = true; + } catch (ConditionTimeoutException e) { + if (expectMigrateSuccess) { + LOGGER.error("Region migrate failed", e); + Assert.fail(); + } + } + if (!expectMigrateSuccess && success) { + LOGGER.error("Region migrate succeeded unexpectedly"); + Assert.fail(); + } + + // make sure all kill points have been triggered + checkKillPointsAllTriggered(configNodeKeywords); + checkKillPointsAllTriggered(dataNodeKeywords); + + // check the remaining file + if (success) { + checkRegionFileClearIfNodeAlive(originalDataNode); + checkRegionFileExistIfNodeAlive(destDataNode); + checkClusterStillWritable(); + } else { + checkRegionFileClearIfNodeAlive(destDataNode); + checkRegionFileExistIfNodeAlive(originalDataNode); + } + + LOGGER.info("test pass"); + } + } + + protected Set getAllDataNodes(Statement statement) throws Exception { + ResultSet result = statement.executeQuery(SHOW_DATANODES); + Set allDataNodeId = new HashSet<>(); + while (result.next()) { + allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); + } + return allDataNodeId; + } + + private void setConfigNodeKillPoints( + KeySetView killConfigNodeKeywords, Consumer action) { + EnvFactory.getEnv() + .getConfigNodeWrapperList() + .forEach( + configNodeWrapper -> + executorService.submit( + () -> + doActionWhenDetectKeywords( + configNodeWrapper, killConfigNodeKeywords, action))); + } + + private void setDataNodeKillPoints( + List dataNodeWrappers, + KeySetView killDataNodeKeywords, + Consumer action) { + dataNodeWrappers.forEach( + dataNodeWrapper -> + executorService.submit( + () -> doActionWhenDetectKeywords(dataNodeWrapper, killDataNodeKeywords, action))); + } + + /** + * Monitor the node's log and kill it when detect specific log. + * + * @param nodeWrapper Easy to understand + * @param keywords When detect these keywords in node's log, stop the node forcibly + */ + private static void doActionWhenDetectKeywords( + AbstractNodeWrapper nodeWrapper, + KeySetView keywords, + Consumer action) { + if (keywords.isEmpty()) { + return; + } + final String logFileName; + if (nodeWrapper instanceof ConfigNodeWrapper) { + logFileName = "log_confignode_all.log"; + } else { + logFileName = "log_datanode_all.log"; + } + SystemType type = SystemType.getSystemType(); + ProcessBuilder builder; + if (type == SystemType.LINUX || type == SystemType.MAC) { + builder = + new ProcessBuilder( + "tail", + "-f", + nodeWrapper.getNodePath() + File.separator + "logs" + File.separator + logFileName); + } else if (type == SystemType.WINDOWS) { + builder = + new ProcessBuilder( + "powershell", + "-Command", + "Get-Content " + + nodeWrapper.getNodePath() + + File.separator + + "logs" + + File.separator + + logFileName + + " -Wait"); + } else { + throw new UnsupportedOperationException("Unsupported system type " + type); + } + builder.redirectErrorStream(true); + + try { + Process process = builder.start(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + // if trigger more than one keyword at a same time, test code may have mistakes + Assert.assertTrue( + line, + keywords.stream().map(KillPoint::addKillPointPrefix).filter(line::contains).count() + <= 1); + String finalLine = line; + Optional detectedKeyword = + keywords.stream() + .filter(keyword -> finalLine.contains(KillPoint.addKillPointPrefix(keyword))) + .findAny(); + if (detectedKeyword.isPresent()) { + // each keyword only trigger once + keywords.remove(detectedKeyword.get()); + action.accept(new KillPointContext(nodeWrapper, (AbstractEnv) EnvFactory.getEnv())); + LOGGER.info("Kill point triggered: {}", detectedKeyword.get()); + } + if (keywords.isEmpty()) { + break; + } + } + } catch (AssertionError e) { + LOGGER.error("gg", e); + throw e; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + void checkKillPointsAllTriggered(KeySetView killPoints) { + if (!killPoints.isEmpty()) { + killPoints.forEach(killPoint -> LOGGER.error("Kill point {} not triggered", killPoint)); + Assert.fail("Some kill points was not triggered"); + } + } + + private static String buildRegionMigrateCommand(int who, int from, int to) { + String result = String.format(REGION_MIGRATE_COMMAND_FORMAT, who, from, to); + LOGGER.info(result); + return result; + } + + public static Map> getDataRegionMap(Statement statement) throws Exception { + ResultSet showRegionsResult = statement.executeQuery(SHOW_REGIONS); + Map> regionMap = new HashMap<>(); + while (showRegionsResult.next()) { + if (String.valueOf(TConsensusGroupType.DataRegion) + .equals(showRegionsResult.getString(ColumnHeaderConstant.TYPE))) { + int regionId = showRegionsResult.getInt(ColumnHeaderConstant.REGION_ID); + int dataNodeId = showRegionsResult.getInt(ColumnHeaderConstant.DATA_NODE_ID); + regionMap.computeIfAbsent(regionId, id -> new HashSet<>()).add(dataNodeId); + } + } + return regionMap; + } + + public static Map> getAllRegionMap(Statement statement) throws Exception { + ResultSet showRegionsResult = statement.executeQuery(SHOW_REGIONS); + Map> regionMap = new HashMap<>(); + while (showRegionsResult.next()) { + int regionId = showRegionsResult.getInt(ColumnHeaderConstant.REGION_ID); + int dataNodeId = showRegionsResult.getInt(ColumnHeaderConstant.DATA_NODE_ID); + regionMap.computeIfAbsent(regionId, id -> new HashSet<>()).add(dataNodeId); + } + return regionMap; + } + + protected static Map> getRegionMap(List regionInfoList) { + Map> regionMap = new HashMap<>(); + regionInfoList.forEach( + regionInfo -> { + int regionId = regionInfo.getConsensusGroupId().getId(); + regionMap + .computeIfAbsent(regionId, regionId1 -> new HashSet<>()) + .add(regionInfo.getDataNodeId()); + }); + return regionMap; + } + + protected static Map> getRunningRegionMap( + List regionInfoList) { + Map> regionMap = new HashMap<>(); + regionInfoList.stream() + .filter(regionInfo -> RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) + .forEach( + regionInfo -> { + int regionId = regionInfo.getConsensusGroupId().getId(); + regionMap + .computeIfAbsent(regionId, regionId1 -> new HashSet<>()) + .add(regionInfo.getDataNodeId()); + }); + return regionMap; + } + + protected static int selectRegion(Map> regionMap) { + return regionMap.keySet().stream().findAny().orElseThrow(() -> new RuntimeException("gg")); + } + + private static int selectOriginalDataNode( + Map> regionMap, int selectedRegion) { + return regionMap.get(selectedRegion).stream() + .findAny() + .orElseThrow(() -> new RuntimeException("cannot find original DataNode")); + } + + protected static int selectDataNodeNotContainsRegion( + Set dataNodeSet, Map> regionMap, int selectedRegion) { + return dataNodeSet.stream() + .filter(dataNodeId -> !regionMap.get(selectedRegion).contains(dataNodeId)) + .findAny() + .orElseThrow(() -> new RuntimeException("cannot find dest DataNode")); + } + + protected static int selectDataNodeContainsRegion( + Set dataNodeSet, Map> regionMap, int selectedRegion) { + return dataNodeSet.stream() + .filter(dataNodeId -> regionMap.get(selectedRegion).contains(dataNodeId)) + .findAny() + .orElseThrow(() -> new RuntimeException("cannot find dest DataNode")); + } + + // I believe this function is not necessary, just keep it here in case it's necessary + private static void awaitUntilFlush(Statement statement, int originalDataNode) throws Exception { + long startTime = System.currentTimeMillis(); + File sequence = new File(buildDataPath(originalDataNode, true)); + File unsequence = new File(buildDataPath(originalDataNode, false)); + Awaitility.await() + .atMost(1, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .until( + () -> { + statement.execute(FLUSH_COMMAND); + int fileNum = 0; + if (sequence.exists() && sequence.listFiles() != null) { + fileNum += Objects.requireNonNull(sequence.listFiles()).length; + } + if (unsequence.exists() && unsequence.listFiles() != null) { + fileNum += Objects.requireNonNull(unsequence.listFiles()).length; + } + return fileNum > 0; + }); + LOGGER.info("DataNode {} has been flushed", originalDataNode); + LOGGER.info("Flush cost time: {}ms", System.currentTimeMillis() - startTime); + } + + protected static void awaitUntilSuccess( + SyncConfigNodeIServiceClient client, + Predicate predicate, + Optional dataNodeExpectInRegionGroup, + Optional dataNodeExpectNotInRegionGroup) { + AtomicReference> lastTimeDataNodes = new AtomicReference<>(); + AtomicReference lastException = new AtomicReference<>(); + AtomicReference clientRef = new AtomicReference<>(client); + try { + Awaitility.await() + .atMost(2, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .until( + () -> { + try { + TShowRegionResp resp = clientRef.get().showRegion(new TShowRegionReq()); + return predicate.test(resp); + } catch (TException e) { + clientRef.set( + (SyncConfigNodeIServiceClient) + EnvFactory.getEnv().getLeaderConfigNodeConnection()); + lastException.set(e); + return false; + } catch (Exception e) { + // Any exception can be ignored + lastException.set(e); + return false; + } + }); + } catch (ConditionTimeoutException e) { + if (lastTimeDataNodes.get() == null) { + LOGGER.error( + "maybe show regions fail, lastTimeDataNodes is null, last Exception:", + lastException.get()); + throw e; + } + String actualSetStr = lastTimeDataNodes.get().toString(); + dataNodeExpectNotInRegionGroup.ifPresent(x -> lastTimeDataNodes.get().remove(x)); + dataNodeExpectInRegionGroup.ifPresent(x -> lastTimeDataNodes.get().add(x)); + String expectSetStr = lastTimeDataNodes.toString(); + LOGGER.error("DataNode Set {} is unexpected, expect {}", actualSetStr, expectSetStr); + if (lastException.get() == null) { + LOGGER.info("No exception during awaiting"); + } else { + LOGGER.error("Last exception during awaiting:", lastException.get()); + } + throw e; + } + LOGGER.info("DataNode set has been successfully changed to {}", lastTimeDataNodes.get()); + } + + private static void checkRegionFileExistIfNodeAlive(int dataNode) { + if (EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().isAlive()) { + checkRegionFileExist(dataNode); + } + } + + private static void checkRegionFileExist(int dataNode) { + File originalRegionDir = new File(buildRegionDirPath(dataNode)); + Assert.assertTrue(originalRegionDir.isDirectory()); + Assert.assertNotEquals(0, Objects.requireNonNull(originalRegionDir.listFiles()).length); + } + + private static void checkRegionFileClearIfNodeAlive(int dataNode) { + if (EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().isAlive()) { + checkRegionFileClear(dataNode); + } + } + + /** Check whether the original DataNode's region file has been deleted. */ + private static void checkRegionFileClear(int dataNode) { + File originalRegionDir = new File(buildRegionDirPath(dataNode)); + Assert.assertTrue(originalRegionDir.isDirectory()); + try { + Assert.assertEquals(0, Objects.requireNonNull(originalRegionDir.listFiles()).length); + } catch (AssertionError e) { + LOGGER.error( + "Original DataNode {} region file not clear, these files is still remain: {}", + dataNode, + Arrays.toString(originalRegionDir.listFiles())); + throw e; + } + LOGGER.info("Original DataNode {} region file clear", dataNode); + } + + private void checkClusterStillWritable() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // check old data + ResultSet resultSet = statement.executeQuery(COUNT_TIMESERIES); + resultSet.next(); + Assert.assertEquals(1, resultSet.getLong(1)); + Assert.assertEquals(1, resultSet.getLong(2)); + LOGGER.info("Old data is still remain"); + // write new data + statement.execute(INSERTION2); + resultSet = statement.executeQuery(COUNT_TIMESERIES); + resultSet.next(); + Assert.assertEquals(2, resultSet.getLong(1)); + Assert.assertEquals(2, resultSet.getLong(2)); + LOGGER.info("Region group is still writable"); + } catch (SQLException e) { + LOGGER.error("Something wrong", e); + Assert.fail("Something wrong"); + } + } + + private static String buildRegionDirPath(int dataNode) { + String nodePath = EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().getNodePath(); + return nodePath + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + "datanode" + + File.separator + + IoTDBConstant.CONSENSUS_FOLDER_NAME + + File.separator + + IoTDBConstant.DATA_REGION_FOLDER_NAME; + } + + private static String buildDataPath(int dataNode, boolean isSequence) { + String nodePath = EnvFactory.getEnv().dataNodeIdToWrapper(dataNode).get().getNodePath(); + return nodePath + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + "datanode" + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + (isSequence ? IoTDBConstant.SEQUENCE_FOLDER_NAME : IoTDBConstant.UNSEQUENCE_FOLDER_NAME); + } + + protected static KeySetView noKillPoints() { + return ConcurrentHashMap.newKeySet(); + } + + @SafeVarargs + protected static > KeySetView buildSet(T... keywords) { + KeySetView result = ConcurrentHashMap.newKeySet(); + result.addAll( + Arrays.stream(keywords).map(KillPoint::enumToString).collect(Collectors.toList())); + return result; + } + + protected static Map getRegionStatusWithoutRunning(Session session) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = session.executeQueryStatement("show regions"); + final int regionIdIndex = dataSet.getColumnNames().indexOf("RegionId"); + final int regionStatusIndex = dataSet.getColumnNames().indexOf("Status"); + dataSet.setFetchSize(1024); + Map result = new TreeMap<>(); + while (dataSet.hasNext()) { + List fields = dataSet.next().getFields(); + final int regionId = fields.get(regionIdIndex).getIntV(); + final String regionStatus = fields.get(regionStatusIndex).toString(); + if (!"Running".equals(regionStatus)) { + result.putIfAbsent(regionId, regionStatus); + } + } + return result; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java deleted file mode 100644 index da6bf2dfdc6f1..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass; - -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework; -import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; -import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.DailyIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@Category({DailyIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateClusterCrashIT extends IoTDBRegionMigrateReliabilityITFramework { - - @Test - public void clusterCrash1() throws Exception { - killClusterTest(buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), true); - } - - @Test - public void clusterCrash2() throws Exception { - killClusterTest(buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), false); - } - - @Test - public void clusterCrash3() throws Exception { - killClusterTest(buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), true); - } - - @Test - public void clusterCrash4() throws Exception { - killClusterTest(buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), true); - } - - @Test - public void clusterCrash6() throws Exception { - killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), true); - } - - @Test - public void clusterCrash7() throws Exception { - killClusterTest(buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), true); - } - - @Test - public void clusterCrash8() throws Exception { - killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), true); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java deleted file mode 100644 index f6916eff40282..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass; - -import org.apache.iotdb.commons.utils.KillPoint.KillNode; -import org.apache.iotdb.commons.utils.KillPoint.KillPoint; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework; -import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; -import org.apache.iotdb.confignode.procedure.state.RegionTransitionState; -import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.DailyIT; - -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -@Category({DailyIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateConfigNodeCrashIT extends IoTDBRegionMigrateReliabilityITFramework { - @Test - @Ignore - public void cnCrashDuringPreCheckTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(RegionTransitionState.REGION_MIGRATE_PREPARE), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashDuringCreatePeerTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void testCnCrashDuringDoAddPeer() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashDuringUpdateCacheTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashDuringChangeRegionLeaderTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashDuringRemoveRegionPeerTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashDuringDeleteOldRegionPeerTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashDuringRemoveRegionLocationCacheTest() throws Exception { - successTest( - 1, - 1, - 1, - 2, - buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), - noKillPoints(), - KillNode.CONFIG_NODE); - } - - @Test - public void cnCrashTest() throws Exception { - ConcurrentHashMap.KeySetView killConfigNodeKeywords = noKillPoints(); - killConfigNodeKeywords.addAll( - Arrays.stream(AddRegionPeerState.values()) - .map(KillPoint::enumToString) - .collect(Collectors.toList())); - killConfigNodeKeywords.addAll( - Arrays.stream(RemoveRegionPeerState.values()) - .map(KillPoint::enumToString) - .collect(Collectors.toList())); - successTest(1, 1, 1, 2, killConfigNodeKeywords, noKillPoints(), KillNode.CONFIG_NODE); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java deleted file mode 100644 index f56296b9cfaea..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass; - -import org.apache.iotdb.commons.utils.KillPoint.KillNode; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@Category({ClusterIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateNormalIT extends IoTDBRegionMigrateReliabilityITFramework { - @Test - public void normal1C2DTest() throws Exception { - successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(), KillNode.ALL_NODES); - } - - @Test - public void normal3C3DTest() throws Exception { - successTest(2, 3, 3, 3, noKillPoints(), noKillPoints(), KillNode.ALL_NODES); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java deleted file mode 100644 index f4ca461edd8b6..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass; - -import org.apache.iotdb.commons.utils.KillPoint.KillNode; -import org.apache.iotdb.commons.utils.KillPoint.NeverTriggeredKillPoint; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@RunWith(IoTDBTestRunner.class) -@Category({ClusterIT.class}) -public class IoTDBRegionMigrateOtherIT extends IoTDBRegionMigrateReliabilityITFramework { - @Test - public void badKillPoint() throws Exception { - try { - successTest( - 1, - 1, - 1, - 2, - buildSet(NeverTriggeredKillPoint.NEVER_TRIGGERED_KILL_POINT), - noKillPoints(), - KillNode.ALL_NODES); - } catch (AssertionError e) { - return; - } - Assert.fail("kill point not triggered but test pass"); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java new file mode 100644 index 0000000000000..3c4aa16a401e4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.commit; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; + +@Category({ClusterIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionGroupExpandAndShrinkForIoTV1IT + extends IoTDBRegionOperationReliabilityITFramework { + private static final String EXPAND_FORMAT = "extend region %d to %d"; + private static final String SHRINK_FORMAT = "remove region %d from %d"; + + private static Logger LOGGER = + LoggerFactory.getLogger(IoTDBRegionGroupExpandAndShrinkForIoTV1IT.class); + + /** + * 1. Expand: {a} -> {a,b} -> ... -> {a,b,c,d,e} + * + *

2. Check + * + *

3. Shrink: {a,b,c,d,e} -> {a,c,d,e} -> ... -> {d} + * + *

4. Check + */ + @Test + public void normal1C5DTest() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataReplicationFactor(1) + .setSchemaReplicationFactor(1); + + EnvFactory.getEnv().initClusterEnvironment(1, 5); + + try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement()); + SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + // prepare data + statement.execute(INSERTION1); + statement.execute(FLUSH_COMMAND); + + // collect necessary information + Map> regionMap = getAllRegionMap(statement); + Set allDataNodeId = getAllDataNodes(statement); + + // expect one data region, one schema region + Assert.assertEquals(2, regionMap.size()); + + // expand + for (int selectedRegion : regionMap.keySet()) { + for (int i = 0; i < 4; i++) { + int targetDataNode = + selectDataNodeNotContainsRegion(allDataNodeId, regionMap, selectedRegion); + regionGroupExpand(statement, client, selectedRegion, targetDataNode); + // update regionMap every time + regionMap = getAllRegionMap(statement); + } + } + + // shrink + for (int selectedRegion : regionMap.keySet()) { + for (int i = 0; i < 4; i++) { + int targetDataNode = + selectDataNodeContainsRegion(allDataNodeId, regionMap, selectedRegion); + regionGroupShrink(statement, client, selectedRegion, targetDataNode); + // update regionMap every time + regionMap = getAllRegionMap(statement); + } + } + } + } + + private void regionGroupExpand( + Statement statement, + SyncConfigNodeIServiceClient client, + int selectedRegion, + int targetDataNode) + throws Exception { + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until( + () -> { + statement.execute(String.format(EXPAND_FORMAT, selectedRegion, targetDataNode)); + return true; + }); + + Predicate expandRegionPredicate = + tShowRegionResp -> { + Map> newRegionMap = + getRunningRegionMap(tShowRegionResp.getRegionInfoList()); + Set dataNodes = newRegionMap.get(selectedRegion); + return dataNodes.contains(targetDataNode); + }; + + awaitUntilSuccess(client, expandRegionPredicate, Optional.of(targetDataNode), Optional.empty()); + + LOGGER.info("Region {} has expanded to DataNode {}", selectedRegion, targetDataNode); + } + + private void regionGroupShrink( + Statement statement, + SyncConfigNodeIServiceClient client, + int selectedRegion, + int targetDataNode) + throws Exception { + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until( + () -> { + statement.execute(String.format(SHRINK_FORMAT, selectedRegion, targetDataNode)); + return true; + }); + + Predicate shrinkRegionPredicate = + tShowRegionResp -> { + Map> newRegionMap = + getRegionMap(tShowRegionResp.getRegionInfoList()); + Set dataNodes = newRegionMap.get(selectedRegion); + return !dataNodes.contains(targetDataNode); + }; + + awaitUntilSuccess(client, shrinkRegionPredicate, Optional.empty(), Optional.of(targetDataNode)); + + LOGGER.info("Region {} has shrunk from DataNode {}", selectedRegion, targetDataNode); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionReconstructForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionReconstructForIoTV1IT.java new file mode 100644 index 0000000000000..05f9289383980 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionReconstructForIoTV1IT.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.commit; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.read.common.RowRecord; +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.sql.Connection; +import java.sql.Statement; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; + +@Category({ClusterIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionReconstructForIoTV1IT extends IoTDBRegionOperationReliabilityITFramework { + private static final String RECONSTRUCT_FORMAT = "reconstruct region %d on %d"; + private static Logger LOGGER = LoggerFactory.getLogger(IoTDBRegionReconstructForIoTV1IT.class); + + @Test + public void normal1C3DTest() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataReplicationFactor(2) + .setSchemaReplicationFactor(3); + + EnvFactory.getEnv().initClusterEnvironment(1, 3); + + try (Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection()); + Statement statement = makeItCloseQuietly(connection.createStatement()); + SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + // prepare data + statement.execute(INSERTION1); + statement.execute(FLUSH_COMMAND); + + // collect necessary information + Map> dataRegionMap = getDataRegionMap(statement); + Set allDataNodeId = getAllDataNodes(statement); + + // select datanode + final int selectedRegion = 1; + Assert.assertTrue(dataRegionMap.containsKey(selectedRegion)); + Assert.assertEquals(2, dataRegionMap.get(selectedRegion).size()); + Iterator iterator = dataRegionMap.get(selectedRegion).iterator(); + final int dataNodeToBeClosed = iterator.next(); + final int dataNodeToBeReconstructed = iterator.next(); + final int dataNodeAlwaysGood = + allDataNodeId.stream() + .filter(x -> x != dataNodeToBeReconstructed && x != dataNodeToBeClosed) + .findAny() + .get(); + final DataNodeWrapper dataNodeWrapper = + EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeAlwaysGood).get(); + Session session = + new Session.Builder() + .host(dataNodeWrapper.getIp()) + .port(dataNodeWrapper.getPort()) + .build(); + session.open(); + + // delete one DataNode's data dir, stop another DataNode + File dataDirToBeReconstructed = + new File( + EnvFactory.getEnv() + .dataNodeIdToWrapper(dataNodeToBeReconstructed) + .get() + .getDataPath()); + FileUtils.deleteDirectory(dataDirToBeReconstructed); + EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeToBeClosed).get().stopForcibly(); + + // now, the query should throw exception + Assert.assertThrows( + StatementExecutionException.class, + () -> session.executeQueryStatement("select * from root.**")); + + // start DataNode, reconstruct the delete one + EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeToBeClosed).get().start(); + EnvFactory.getAbstractEnv().checkNodeInStatus(dataNodeToBeClosed, NodeStatus.Running); + session.executeNonQueryStatement( + String.format(RECONSTRUCT_FORMAT, selectedRegion, dataNodeToBeReconstructed)); + try { + Awaitility.await() + .pollInterval(1, TimeUnit.SECONDS) + .atMost(1, TimeUnit.MINUTES) + .until( + () -> + getRegionStatusWithoutRunning(session).isEmpty() + && dataDirToBeReconstructed.getAbsoluteFile().exists()); + } catch (Exception e) { + LOGGER.error( + "Two factor: {} && {}", + getRegionStatusWithoutRunning(session).isEmpty(), + dataDirToBeReconstructed.getAbsoluteFile().exists()); + Assert.fail(); + } + EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeToBeClosed).get().stopForcibly(); + + // now, the query should work fine + SessionDataSet resultSet = session.executeQueryStatement("select * from root.**"); + RowRecord rowRecord = resultSet.next(); + Assert.assertEquals("2.0", rowRecord.getField(0).getStringValue()); + Assert.assertEquals("1.0", rowRecord.getField(1).getStringValue()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java new file mode 100644 index 0000000000000..b763b4240028a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.commit.batch; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({ClusterIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateNormalITForIoTV2BatchIT + extends IoTDBRegionOperationReliabilityITFramework { + @Test + public void normal1C2DTest() throws Exception { + successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(), KillNode.ALL_NODES); + } + + @Test + public void normal3C3DTest() throws Exception { + successTest(2, 3, 3, 3, noKillPoints(), noKillPoints(), KillNode.ALL_NODES); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateOtherForIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateOtherForIoTV2BatchIT.java new file mode 100644 index 0000000000000..f3c2b089fd268 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateOtherForIoTV2BatchIT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.commit.batch; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.commons.utils.KillPoint.NeverTriggeredKillPoint; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBRegionMigrateOtherForIoTV2BatchIT + extends IoTDBRegionOperationReliabilityITFramework { + @Test + public void badKillPoint() throws Exception { + try { + successTest( + 1, + 1, + 1, + 2, + buildSet(NeverTriggeredKillPoint.NEVER_TRIGGERED_KILL_POINT), + noKillPoints(), + KillNode.ALL_NODES); + } catch (AssertionError e) { + return; + } + Assert.fail("kill point not triggered but test pass"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/stream/IoTDBRegionMigrateNormalForIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/stream/IoTDBRegionMigrateNormalForIoTV2StreamIT.java new file mode 100644 index 0000000000000..aaaa4dadacbde --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/stream/IoTDBRegionMigrateNormalForIoTV2StreamIT.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.commit.stream; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({ClusterIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateNormalForIoTV2StreamIT + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void normal1C2DTest() throws Exception { + successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(), KillNode.ALL_NODES); + } + + @Test + public void normal3C3DTest() throws Exception { + successTest(2, 3, 3, 3, noKillPoints(), noKillPoints(), KillNode.ALL_NODES); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/stream/IoTDBRegionMigrateOtherITForIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/stream/IoTDBRegionMigrateOtherITForIoTV2StreamIT.java new file mode 100644 index 0000000000000..a8db27850ab8e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/stream/IoTDBRegionMigrateOtherITForIoTV2StreamIT.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.commit.stream; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.commons.utils.KillPoint.NeverTriggeredKillPoint; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBRegionMigrateOtherITForIoTV2StreamIT + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void badKillPoint() throws Exception { + try { + successTest( + 1, + 1, + 1, + 2, + buildSet(NeverTriggeredKillPoint.NEVER_TRIGGERED_KILL_POINT), + noKillPoints(), + KillNode.ALL_NODES); + } catch (AssertionError e) { + return; + } + Assert.fail("kill point not triggered but test pass"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV1IT.java new file mode 100644 index 0000000000000..9f37286136a02 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV1IT.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv1; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusRemovePeerCoordinatorKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV1IT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1 { + + @Test + public void initCrash() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.INIT); + } + + @Test + public void crashAfterNotifyPeersToRemoveSyncLogChannel() throws Exception { + success( + IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_NOTIFY_PEERS_TO_REMOVE_REPLICATE_CHANNEL); + } + + @Test + public void crashAfterInactivePeer() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_INACTIVE_PEER); + } + + @Test + public void crashAfterFinish() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.FINISH); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateDataNodeCrashForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateDataNodeCrashForIoTV1IT.java new file mode 100644 index 0000000000000..50beb4ec5493c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateDataNodeCrashForIoTV1IT.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv1; + +import org.apache.iotdb.commons.utils.KillPoint.DataNodeKillPoints; +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateDataNodeCrashForIoTV1IT + extends IoTDBRegionOperationReliabilityITFramework { + // region Coordinator DataNode crash tests + + private final int dataReplicateFactor = 2; + private final int schemaReplicationFactor = 2; + private final int configNodeNum = 1; + private final int dataNodeNum = 3; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + } + + @Test + public void coordinatorCrashDuringAddPeerTransition() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_TRANSITION), + KillNode.COORDINATOR_DATANODE); + } + + @Test + public void coordinatorCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_DONE), + KillNode.COORDINATOR_DATANODE); + } + + // endregion ---------------------------------------------- + + // region Original DataNode crash tests + + @Test + public void originalCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.ORIGINAL_ADD_PEER_DONE), + KillNode.ORIGINAL_DATANODE); + } + + // endregion ---------------------------------------------- + + // region Destination DataNode crash tests + + @Test + public void destinationCrashDuringCreateLocalPeer() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_CREATE_LOCAL_PEER), + KillNode.DESTINATION_DATANODE); + } + + @Test + public void destinationCrashDuringAddPeerTransition() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_ADD_PEER_TRANSITION), + KillNode.DESTINATION_DATANODE); + } + + @Test + public void destinationCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_ADD_PEER_DONE), + KillNode.DESTINATION_DATANODE); + } + + // endregion ---------------------------------------------- +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV1IT.java new file mode 100644 index 0000000000000..219e036999db7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV1IT.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv1; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusDeleteLocalPeerKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV1IT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1 { + @Test + public void crashBeforeDelete() throws Exception { + success(IoTConsensusDeleteLocalPeerKillPoints.BEFORE_DELETE); + } + + @Test + public void crashAfterDelete() throws Exception { + success(IoTConsensusDeleteLocalPeerKillPoints.AFTER_DELETE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV1IT.java new file mode 100644 index 0000000000000..40301b3ab3be9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv1/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV1IT.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv1; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusInactivatePeerKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV1IT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV1 { + @Test + public void crashBeforeInactivate() throws Exception { + success(IoTConsensusInactivatePeerKillPoints.BEFORE_INACTIVATE); + } + + @Test + public void crashAfterInactivate() throws Exception { + success(IoTConsensusInactivatePeerKillPoints.AFTER_INACTIVATE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2BatchIT.java new file mode 100644 index 0000000000000..7cf0f516ef05a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2BatchIT.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.batch; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusRemovePeerCoordinatorKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2BatchIT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 { + + @Test + public void initCrash() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.INIT); + } + + @Test + public void crashAfterNotifyPeersToRemoveSyncLogChannel() throws Exception { + success( + IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_NOTIFY_PEERS_TO_REMOVE_REPLICATE_CHANNEL); + } + + @Test + public void crashAfterInactivePeer() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_INACTIVE_PEER); + } + + @Test + public void crashAfterFinish() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.FINISH); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateDataNodeCrashForIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateDataNodeCrashForIoTV2BatchIT.java new file mode 100644 index 0000000000000..1fea8b70a6fed --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateDataNodeCrashForIoTV2BatchIT.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.batch; + +import org.apache.iotdb.commons.utils.KillPoint.DataNodeKillPoints; +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateDataNodeCrashForIoTV2BatchIT + extends IoTDBRegionOperationReliabilityITFramework { + // region Coordinator DataNode crash tests + + private final int dataReplicateFactor = 2; + private final int schemaReplicationFactor = 2; + private final int configNodeNum = 1; + private final int dataNodeNum = 3; + + @Test + public void coordinatorCrashDuringAddPeerTransition() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_TRANSITION), + KillNode.COORDINATOR_DATANODE); + } + + @Test + public void coordinatorCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_DONE), + KillNode.COORDINATOR_DATANODE); + } + + // endregion ---------------------------------------------- + + // region Original DataNode crash tests + + @Test + public void originalCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.ORIGINAL_ADD_PEER_DONE), + KillNode.ORIGINAL_DATANODE); + } + + // endregion ---------------------------------------------- + + // region Destination DataNode crash tests + + @Test + public void destinationCrashDuringCreateLocalPeer() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_CREATE_LOCAL_PEER), + KillNode.DESTINATION_DATANODE); + } + + @Test + public void destinationCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_ADD_PEER_DONE), + KillNode.DESTINATION_DATANODE); + } + + // endregion ---------------------------------------------- +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2BatchIT.java new file mode 100644 index 0000000000000..e0273491ccb80 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2BatchIT.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.batch; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusDeleteLocalPeerKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2BatchIT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 { + @Test + public void crashBeforeDelete() throws Exception { + success(IoTConsensusDeleteLocalPeerKillPoints.BEFORE_DELETE); + } + + @Test + public void crashAfterDelete() throws Exception { + success(IoTConsensusDeleteLocalPeerKillPoints.AFTER_DELETE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2BatchIT.java new file mode 100644 index 0000000000000..5d7f13664d67d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/batch/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2BatchIT.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.batch; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusInactivatePeerKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2BatchIT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 { + @Test + public void crashBeforeInactivate() throws Exception { + success(IoTConsensusInactivatePeerKillPoints.BEFORE_INACTIVATE); + } + + @Test + public void crashAfterInactivate() throws Exception { + success(IoTConsensusInactivatePeerKillPoints.AFTER_INACTIVATE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2StreamIT.java new file mode 100644 index 0000000000000..6d4e0e223cc75 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2StreamIT.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.stream; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusRemovePeerCoordinatorKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerForIoTV2StreamIT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void initCrash() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.INIT); + } + + @Test + public void crashAfterNotifyPeersToRemoveSyncLogChannel() throws Exception { + success( + IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_NOTIFY_PEERS_TO_REMOVE_REPLICATE_CHANNEL); + } + + @Test + public void crashAfterInactivePeer() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_INACTIVE_PEER); + } + + @Test + public void crashAfterFinish() throws Exception { + success(IoTConsensusRemovePeerCoordinatorKillPoints.FINISH); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateDataNodeCrashForIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateDataNodeCrashForIoTV2StreamIT.java new file mode 100644 index 0000000000000..4c21b63330e49 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateDataNodeCrashForIoTV2StreamIT.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.stream; + +import org.apache.iotdb.commons.utils.KillPoint.DataNodeKillPoints; +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateDataNodeCrashForIoTV2StreamIT + extends IoTDBRegionOperationReliabilityITFramework { + // region Coordinator DataNode crash tests + + private final int dataReplicateFactor = 2; + private final int schemaReplicationFactor = 2; + private final int configNodeNum = 1; + private final int dataNodeNum = 3; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void coordinatorCrashDuringAddPeerTransition() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_TRANSITION), + KillNode.COORDINATOR_DATANODE); + } + + @Test + public void coordinatorCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_DONE), + KillNode.COORDINATOR_DATANODE); + } + + // endregion ---------------------------------------------- + + // region Original DataNode crash tests + + @Test + public void originalCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.ORIGINAL_ADD_PEER_DONE), + KillNode.ORIGINAL_DATANODE); + } + + // endregion ---------------------------------------------- + + // region Destination DataNode crash tests + + @Test + public void destinationCrashDuringCreateLocalPeer() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_CREATE_LOCAL_PEER), + KillNode.DESTINATION_DATANODE); + } + + @Test + public void destinationCrashDuringAddPeerDone() throws Exception { + failTest( + 2, + 2, + 1, + 3, + noKillPoints(), + buildSet(DataNodeKillPoints.DESTINATION_ADD_PEER_DONE), + KillNode.DESTINATION_DATANODE); + } + + // endregion ---------------------------------------------- +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2StreamIT.java new file mode 100644 index 0000000000000..f7cfc6995c500 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2StreamIT.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.stream; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusDeleteLocalPeerKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerForIoTV2StreamIT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void crashBeforeDelete() throws Exception { + success(IoTConsensusDeleteLocalPeerKillPoints.BEFORE_DELETE); + } + + @Test + public void crashAfterDelete() throws Exception { + success(IoTConsensusDeleteLocalPeerKillPoints.AFTER_DELETE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2StreamIT.java new file mode 100644 index 0000000000000..9b38a557a20d7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/datanodecrash/iotv2/stream/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2StreamIT.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.datanodecrash.iotv2.stream; + +import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusInactivatePeerKillPoints; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerForIoTV2StreamIT + extends IoTDBRegionMigrateDataNodeCrashITFrameworkForIoTV2 { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void crashBeforeInactivate() throws Exception { + success(IoTConsensusInactivatePeerKillPoints.BEFORE_INACTIVATE); + } + + @Test + public void crashAfterInactivate() throws Exception { + success(IoTConsensusInactivatePeerKillPoints.AFTER_INACTIVATE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateClusterCrashIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateClusterCrashIoTV1IT.java new file mode 100644 index 0000000000000..544c4b568170d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateClusterCrashIoTV1IT.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1; + +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; +import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateClusterCrashIoTV1IT + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + } + + @Test + public void clusterCrash1() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), true); + } + + @Test + public void clusterCrash2() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), false); + } + + @Test + public void clusterCrash3() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), true); + } + + @Test + public void clusterCrash4() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), true); + } + + @Test + public void clusterCrash6() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), true); + } + + @Test + public void clusterCrash7() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), true); + } + + @Test + public void clusterCrash8() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), true); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateConfigNodeCrashIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateConfigNodeCrashIoTV1IT.java new file mode 100644 index 0000000000000..ebda0c36ee327 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateConfigNodeCrashIoTV1IT.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.commons.utils.KillPoint.KillPoint; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; +import org.apache.iotdb.confignode.procedure.state.RegionTransitionState; +import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateConfigNodeCrashIoTV1IT + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + } + + @Test + @Ignore + public void cnCrashDuringPreCheckTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RegionTransitionState.REGION_MIGRATE_PREPARE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringCreatePeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void testCnCrashDuringDoAddPeer() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringUpdateCacheTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringChangeRegionLeaderTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringRemoveRegionPeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringDeleteOldRegionPeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringRemoveRegionLocationCacheTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashTest() throws Exception { + ConcurrentHashMap.KeySetView killConfigNodeKeywords = noKillPoints(); + killConfigNodeKeywords.addAll( + Arrays.stream(AddRegionPeerState.values()) + .map(KillPoint::enumToString) + .collect(Collectors.toList())); + killConfigNodeKeywords.addAll( + Arrays.stream(RemoveRegionPeerState.values()) + .map(KillPoint::enumToString) + .collect(Collectors.toList())); + successTest(1, 1, 1, 2, killConfigNodeKeywords, noKillPoints(), KillNode.CONFIG_NODE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/batch/IoTDBRegionMigrateClusterCrashIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/batch/IoTDBRegionMigrateClusterCrashIoTV2BatchIT.java new file mode 100644 index 0000000000000..6a74d99c09672 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/batch/IoTDBRegionMigrateClusterCrashIoTV2BatchIT.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv2.batch; + +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; +import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateClusterCrashIoTV2BatchIT + extends IoTDBRegionOperationReliabilityITFramework { + + @Test + public void clusterCrash1() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), true); + } + + @Test + public void clusterCrash2() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), false); + } + + @Test + public void clusterCrash3() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), true); + } + + @Test + public void clusterCrash4() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), true); + } + + @Test + public void clusterCrash6() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), true); + } + + @Test + public void clusterCrash7() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), true); + } + + @Test + public void clusterCrash8() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), true); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/batch/IoTDBRegionMigrateConfigNodeCrashIoTV2BatchIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/batch/IoTDBRegionMigrateConfigNodeCrashIoTV2BatchIT.java new file mode 100644 index 0000000000000..853d834966614 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/batch/IoTDBRegionMigrateConfigNodeCrashIoTV2BatchIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv2.batch; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.commons.utils.KillPoint.KillPoint; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; +import org.apache.iotdb.confignode.procedure.state.RegionTransitionState; +import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateConfigNodeCrashIoTV2BatchIT + extends IoTDBRegionOperationReliabilityITFramework { + @Test + @Ignore + public void cnCrashDuringPreCheckTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RegionTransitionState.REGION_MIGRATE_PREPARE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringCreatePeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void testCnCrashDuringDoAddPeer() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringUpdateCacheTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringChangeRegionLeaderTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringRemoveRegionPeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringDeleteOldRegionPeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringRemoveRegionLocationCacheTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashTest() throws Exception { + ConcurrentHashMap.KeySetView killConfigNodeKeywords = noKillPoints(); + killConfigNodeKeywords.addAll( + Arrays.stream(AddRegionPeerState.values()) + .map(KillPoint::enumToString) + .collect(Collectors.toList())); + killConfigNodeKeywords.addAll( + Arrays.stream(RemoveRegionPeerState.values()) + .map(KillPoint::enumToString) + .collect(Collectors.toList())); + successTest(1, 1, 1, 2, killConfigNodeKeywords, noKillPoints(), KillNode.CONFIG_NODE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/stream/IoTDBRegionMigrateClusterCrashIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/stream/IoTDBRegionMigrateClusterCrashIoTV2StreamIT.java new file mode 100644 index 0000000000000..5f0f2fe3cee4f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/stream/IoTDBRegionMigrateClusterCrashIoTV2StreamIT.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv2.stream; + +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; +import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateClusterCrashIoTV2StreamIT + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + public void clusterCrash1() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), true); + } + + @Test + public void clusterCrash2() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), false); + } + + @Test + public void clusterCrash3() throws Exception { + killClusterTest(buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), true); + } + + @Test + public void clusterCrash4() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), true); + } + + @Test + public void clusterCrash6() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), true); + } + + @Test + public void clusterCrash7() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), true); + } + + @Test + public void clusterCrash8() throws Exception { + killClusterTest(buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), true); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/stream/IoTDBRegionMigrateConfigNodeCrashIoTV2StreamIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/stream/IoTDBRegionMigrateConfigNodeCrashIoTV2StreamIT.java new file mode 100644 index 0000000000000..f29482811f0a0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv2/stream/IoTDBRegionMigrateConfigNodeCrashIoTV2StreamIT.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv2.stream; + +import org.apache.iotdb.commons.utils.KillPoint.KillNode; +import org.apache.iotdb.commons.utils.KillPoint.KillPoint; +import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework; +import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState; +import org.apache.iotdb.confignode.procedure.state.RegionTransitionState; +import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.DailyIT; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Category({DailyIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRegionMigrateConfigNodeCrashIoTV2StreamIT + extends IoTDBRegionOperationReliabilityITFramework { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setIoTConsensusV2Mode(ConsensusFactory.IOT_CONSENSUS_V2_STREAM_MODE); + } + + @Test + @Ignore + public void cnCrashDuringPreCheckTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RegionTransitionState.REGION_MIGRATE_PREPARE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringCreatePeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.CREATE_NEW_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void testCnCrashDuringDoAddPeer() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.DO_ADD_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringUpdateCacheTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(AddRegionPeerState.UPDATE_REGION_LOCATION_CACHE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringChangeRegionLeaderTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.TRANSFER_REGION_LEADER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringRemoveRegionPeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.REMOVE_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringDeleteOldRegionPeerTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.DELETE_OLD_REGION_PEER), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashDuringRemoveRegionLocationCacheTest() throws Exception { + successTest( + 1, + 1, + 1, + 2, + buildSet(RemoveRegionPeerState.REMOVE_REGION_LOCATION_CACHE), + noKillPoints(), + KillNode.CONFIG_NODE); + } + + @Test + public void cnCrashTest() throws Exception { + ConcurrentHashMap.KeySetView killConfigNodeKeywords = noKillPoints(); + killConfigNodeKeywords.addAll( + Arrays.stream(AddRegionPeerState.values()) + .map(KillPoint::enumToString) + .collect(Collectors.toList())); + killConfigNodeKeywords.addAll( + Arrays.stream(RemoveRegionPeerState.values()) + .map(KillPoint::enumToString) + .collect(Collectors.toList())); + successTest(1, 1, 1, 2, killConfigNodeKeywords, noKillPoints(), KillNode.CONFIG_NODE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java deleted file mode 100644 index 11fd4552fec62..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash; - -import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusRemovePeerCoordinatorKillPoints; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.DailyIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@Category({DailyIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT - extends IoTDBRegionMigrateDataNodeCrashITFramework { - - @Test - public void initCrash() throws Exception { - success(IoTConsensusRemovePeerCoordinatorKillPoints.INIT); - } - - @Test - public void crashAfterNotifyPeersToRemoveSyncLogChannel() throws Exception { - success( - IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_NOTIFY_PEERS_TO_REMOVE_SYNC_LOG_CHANNEL); - } - - @Test - public void crashAfterInactivePeer() throws Exception { - success(IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_INACTIVE_PEER); - } - - @Test - public void crashAfterFinish() throws Exception { - success(IoTConsensusRemovePeerCoordinatorKillPoints.FINISH); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java deleted file mode 100644 index 6819af9f87b3f..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash; - -import org.apache.iotdb.commons.utils.KillPoint.DataNodeKillPoints; -import org.apache.iotdb.commons.utils.KillPoint.KillNode; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.DailyIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@Category({DailyIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateDataNodeCrashIT extends IoTDBRegionMigrateReliabilityITFramework { - // region Coordinator DataNode crash tests - - private final int dataReplicateFactor = 2; - private final int schemaReplicationFactor = 2; - private final int configNodeNum = 1; - private final int dataNodeNum = 3; - - @Test - public void coordinatorCrashDuringAddPeerTransition() throws Exception { - failTest( - 2, - 2, - 1, - 3, - noKillPoints(), - buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_TRANSITION), - KillNode.COORDINATOR_DATANODE); - } - - @Test - public void coordinatorCrashDuringAddPeerDone() throws Exception { - failTest( - 2, - 2, - 1, - 3, - noKillPoints(), - buildSet(DataNodeKillPoints.COORDINATOR_ADD_PEER_DONE), - KillNode.COORDINATOR_DATANODE); - } - - // endregion ---------------------------------------------- - - // region Original DataNode crash tests - - @Test - public void originalCrashDuringAddPeerDone() throws Exception { - failTest( - 2, - 2, - 1, - 3, - noKillPoints(), - buildSet(DataNodeKillPoints.ORIGINAL_ADD_PEER_DONE), - KillNode.ORIGINAL_DATANODE); - } - - // endregion ---------------------------------------------- - - // region Destination DataNode crash tests - - @Test - public void destinationCrashDuringCreateLocalPeer() throws Exception { - failTest( - 2, - 2, - 1, - 3, - noKillPoints(), - buildSet(DataNodeKillPoints.DESTINATION_CREATE_LOCAL_PEER), - KillNode.DESTINATION_DATANODE); - } - - @Test - public void destinationCrashDuringAddPeerTransition() throws Exception { - failTest( - 2, - 2, - 1, - 3, - noKillPoints(), - buildSet(DataNodeKillPoints.DESTINATION_ADD_PEER_TRANSITION), - KillNode.DESTINATION_DATANODE); - } - - @Test - public void destinationCrashDuringAddPeerDone() throws Exception { - failTest( - 2, - 2, - 1, - 3, - noKillPoints(), - buildSet(DataNodeKillPoints.DESTINATION_ADD_PEER_DONE), - KillNode.DESTINATION_DATANODE); - } - - // endregion ---------------------------------------------- -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java deleted file mode 100644 index 047037c7d147e..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash; - -import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusDeleteLocalPeerKillPoints; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.DailyIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@Category({DailyIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT - extends IoTDBRegionMigrateDataNodeCrashITFramework { - @Test - public void crashBeforeDelete() throws Exception { - success(IoTConsensusDeleteLocalPeerKillPoints.BEFORE_DELETE); - } - - @Test - public void crashAfterDelete() throws Exception { - success(IoTConsensusDeleteLocalPeerKillPoints.AFTER_DELETE); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java deleted file mode 100644 index 69ea8c27088f1..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash; - -import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusInactivatePeerKillPoints; -import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.DailyIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -@Category({DailyIT.class}) -@RunWith(IoTDBTestRunner.class) -public class IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT - extends IoTDBRegionMigrateDataNodeCrashITFramework { - @Test - public void crashBeforeInactivate() throws Exception { - success(IoTConsensusInactivatePeerKillPoints.BEFORE_INACTIVATE); - } - - @Test - public void crashAfterInactivate() throws Exception { - success(IoTConsensusInactivatePeerKillPoints.AFTER_INACTIVATE); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java new file mode 100644 index 0000000000000..8ba221a32333d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.removeconfignode; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.it.removedatanode.SQLModel; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.exception.InconsistentDataException; +import org.apache.iotdb.jdbc.IoTDBSQLException; +import org.apache.iotdb.relational.it.query.old.aligned.TableUtils; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMap; +import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeITFramework.getConnectionWithSQLType; +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; + +public class IoTDBRemoveConfigNodeITFramework { + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBRemoveConfigNodeITFramework.class); + private static final String TREE_MODEL_INSERTION = + "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(100, 1, 2)"; + + private static final String SHOW_CONFIGNODES = "show confignodes"; + + private static final String defaultSchemaRegionGroupExtensionPolicy = "CUSTOM"; + private static final String defaultDataRegionGroupExtensionPolicy = "CUSTOM"; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaRegionGroupExtensionPolicy(defaultSchemaRegionGroupExtensionPolicy) + .setDataRegionGroupExtensionPolicy(defaultDataRegionGroupExtensionPolicy); + } + + @After + public void tearDown() throws InterruptedException { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + public void testRemoveConfigNode( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + final int dataRegionPerDataNode, + final SQLModel model) + throws Exception { + + // Set up the environment + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaReplicationFactor(schemaReplicationFactor) + .setDataReplicationFactor(dataReplicateFactor) + .setDefaultDataRegionGroupNumPerDatabase( + dataRegionPerDataNode * dataNodeNum / dataReplicateFactor); + EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum); + + try (final Connection connection = makeItCloseQuietly(getConnectionWithSQLType(model)); + final Statement statement = makeItCloseQuietly(connection.createStatement()); + SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + + if (SQLModel.TABLE_MODEL_SQL.equals(model)) { + // Insert data in table model + TableUtils.insertData(); + } else { + // Insert data in tree model + statement.execute(TREE_MODEL_INSERTION); + } + + Map> regionMap = getDataRegionMap(statement); + regionMap.forEach( + (key, valueSet) -> { + LOGGER.info("Key: {}, Value: {}", key, valueSet); + if (valueSet.size() != dataReplicateFactor) { + Assert.fail(); + } + }); + + // Get all config nodes + ResultSet result = statement.executeQuery(SHOW_CONFIGNODES); + Set allConfigNodeId = new HashSet<>(); + while (result.next()) { + allConfigNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); + } + + AtomicReference clientRef = new AtomicReference<>(client); + + int removeConfigNodeId = allConfigNodeId.iterator().next(); + String removeConfigNodeSQL = generateRemoveString(removeConfigNodeId); + LOGGER.info("Remove ConfigNodes SQL: {}", removeConfigNodeSQL); + try { + statement.execute(removeConfigNodeSQL); + } catch (IoTDBSQLException e) { + LOGGER.error("Remove ConfigNodes SQL execute fail: {}", e.getMessage()); + Assert.fail(); + } + LOGGER.info("Remove ConfigNodes SQL submit successfully."); + + // Wait until success + try { + awaitUntilSuccess(statement, removeConfigNodeId); + } catch (ConditionTimeoutException e) { + LOGGER.error("Remove ConfigNodes timeout in 2 minutes"); + Assert.fail(); + } + + LOGGER.info("Remove ConfigNodes success"); + } catch (InconsistentDataException e) { + LOGGER.error("Unexpected error:", e); + } + } + + private static void awaitUntilSuccess(Statement statement, int removeConfigNodeId) { + AtomicReference> lastTimeConfigNodes = new AtomicReference<>(); + AtomicReference lastException = new AtomicReference<>(); + + try { + Awaitility.await() + .atMost(2, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .until( + () -> { + try { + // Get all config nodes + ResultSet result = statement.executeQuery(SHOW_CONFIGNODES); + Set allConfigNodeId = new HashSet<>(); + while (result.next()) { + allConfigNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); + } + lastTimeConfigNodes.set(allConfigNodeId); + return !allConfigNodeId.contains(removeConfigNodeId); + } catch (Exception e) { + // Any exception can be ignored + lastException.set(e); + return false; + } + }); + } catch (ConditionTimeoutException e) { + if (lastTimeConfigNodes.get() == null) { + LOGGER.error( + "Maybe show confignodes fail, lastTimeConfigNodes is null, last Exception:", + lastException.get()); + throw e; + } + String actualSetStr = lastTimeConfigNodes.get().toString(); + lastTimeConfigNodes.get().remove(removeConfigNodeId); + String expectedSetStr = lastTimeConfigNodes.get().toString(); + LOGGER.error( + "Remove ConfigNode timeout in 2 minutes, expected set: {}, actual set: {}", + expectedSetStr, + actualSetStr); + if (lastException.get() == null) { + LOGGER.info("No exception during awaiting"); + } else { + LOGGER.error("Last exception during awaiting:", lastException.get()); + } + throw e; + } + } + + public static String generateRemoveString(Integer configNodeId) { + return "remove confignode " + configNodeId; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java new file mode 100644 index 0000000000000..2b5026f9ad903 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.removeconfignode; + +import org.apache.iotdb.confignode.it.removedatanode.SQLModel; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({ClusterIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRemoveConfigNodeNormalIT extends IoTDBRemoveConfigNodeITFramework { + @Test + public void test3C1DUseTreeSQL() throws Exception { + testRemoveConfigNode(1, 1, 3, 1, 2, SQLModel.TREE_MODEL_SQL); + } + + @Test + public void test3C1DUseTableSQL() throws Exception { + testRemoveConfigNode(1, 1, 3, 1, 2, SQLModel.TABLE_MODEL_SQL); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java new file mode 100644 index 0000000000000..583410d80c52b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java @@ -0,0 +1,417 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.removedatanode; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRemoveReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRemoveResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.itbase.exception.InconsistentDataException; +import org.apache.iotdb.jdbc.IoTDBSQLException; +import org.apache.iotdb.relational.it.query.old.aligned.TableUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMap; +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; + +public class IoTDBRemoveDataNodeITFramework { + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBRemoveDataNodeITFramework.class); + private static final String TREE_MODEL_INSERTION = + "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(100, 1, 2)"; + + private static final String SHOW_REGIONS = "show regions"; + private static final String SHOW_DATANODES = "show datanodes"; + + private static final String defaultSchemaRegionGroupExtensionPolicy = "CUSTOM"; + private static final String defaultDataRegionGroupExtensionPolicy = "CUSTOM"; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaRegionGroupExtensionPolicy(defaultSchemaRegionGroupExtensionPolicy) + .setDataRegionGroupExtensionPolicy(defaultDataRegionGroupExtensionPolicy); + } + + @After + public void tearDown() throws InterruptedException { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + public void successTest( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + final int removeDataNodeNum, + final int dataRegionPerDataNode, + final boolean rejoinRemovedDataNode, + final SQLModel model) + throws Exception { + testRemoveDataNode( + dataReplicateFactor, + schemaReplicationFactor, + configNodeNum, + dataNodeNum, + removeDataNodeNum, + dataRegionPerDataNode, + true, + rejoinRemovedDataNode, + model); + } + + public void failTest( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + final int removeDataNodeNum, + final int dataRegionPerDataNode, + final boolean rejoinRemovedDataNode, + final SQLModel model) + throws Exception { + testRemoveDataNode( + dataReplicateFactor, + schemaReplicationFactor, + configNodeNum, + dataNodeNum, + removeDataNodeNum, + dataRegionPerDataNode, + false, + rejoinRemovedDataNode, + model); + } + + public void testRemoveDataNode( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + final int removeDataNodeNum, + final int dataRegionPerDataNode, + final boolean expectRemoveSuccess, + final boolean rejoinRemovedDataNode, + final SQLModel model) + throws Exception { + // Set up the environment + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaReplicationFactor(schemaReplicationFactor) + .setDataReplicationFactor(dataReplicateFactor) + .setDefaultDataRegionGroupNumPerDatabase( + dataRegionPerDataNode * dataNodeNum / dataReplicateFactor); + EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum); + + try (final Connection connection = makeItCloseQuietly(getConnectionWithSQLType(model)); + final Statement statement = makeItCloseQuietly(connection.createStatement()); + SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + + if (SQLModel.TABLE_MODEL_SQL.equals(model)) { + // Insert data in table model + TableUtils.insertData(); + } else { + // Insert data in tree model + statement.execute(TREE_MODEL_INSERTION); + } + + Map> regionMap = getDataRegionMap(statement); + regionMap.forEach( + (key, valueSet) -> { + LOGGER.info("Key: {}, Value: {}", key, valueSet); + if (valueSet.size() != dataReplicateFactor) { + Assert.fail(); + } + }); + + // Get all data nodes + ResultSet result = statement.executeQuery(SHOW_DATANODES); + Set allDataNodeId = new HashSet<>(); + while (result.next()) { + allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); + } + + // Select data nodes to remove + final Set removeDataNodes = + selectRemoveDataNodes(allDataNodeId, regionMap, removeDataNodeNum); + + List removeDataNodeWrappers = + removeDataNodes.stream() + .map(dataNodeId -> EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeId).get()) + .collect(Collectors.toList()); + + AtomicReference clientRef = new AtomicReference<>(client); + List removeDataNodeLocations = + clientRef + .get() + .getDataNodeConfiguration(-1) + .getDataNodeConfigurationMap() + .values() + .stream() + .map(TDataNodeConfiguration::getLocation) + .filter(location -> removeDataNodes.contains(location.getDataNodeId())) + .collect(Collectors.toList()); + if (SQLModel.NOT_USE_SQL.equals(model)) { + TDataNodeRemoveReq removeReq = new TDataNodeRemoveReq(removeDataNodeLocations); + + // Remove data nodes + TDataNodeRemoveResp removeResp = clientRef.get().removeDataNode(removeReq); + LOGGER.info("Submit Remove DataNodes result {} ", removeResp); + if (removeResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (expectRemoveSuccess) { + LOGGER.error("Submit Remove DataNodes fail"); + Assert.fail(); + } else { + LOGGER.info("Submit Remove DataNodes fail, as expected."); + return; + } + } + LOGGER.info("Submit Remove DataNodes request: {}", removeReq); + + } else { + String removeDataNodeSQL = generateRemoveString(removeDataNodes); + LOGGER.info("Remove DataNodes SQL: {}", removeDataNodeSQL); + try { + statement.execute(removeDataNodeSQL); + } catch (IoTDBSQLException e) { + if (expectRemoveSuccess) { + LOGGER.error("Remove DataNodes SQL execute fail: {}", e.getMessage()); + Assert.fail(); + } else { + LOGGER.info("Submit Remove DataNodes fail, as expected"); + return; + } + } + LOGGER.info("Remove DataNodes SQL submit successfully."); + } + + // Wait until success + boolean removeSuccess = false; + try { + awaitUntilSuccess(clientRef, removeDataNodeLocations); + removeSuccess = true; + } catch (ConditionTimeoutException e) { + if (expectRemoveSuccess) { + LOGGER.error("Remove DataNodes timeout in 2 minutes"); + Assert.fail(); + } + } + + if (!expectRemoveSuccess && removeSuccess) { + LOGGER.error("Remove DataNodes success, but expect fail"); + Assert.fail(); + } + + LOGGER.info("Remove DataNodes success"); + + if (rejoinRemovedDataNode) { + try { + // Use sleep and restart to ensure that removeDataNodes restarts successfully + Thread.sleep(30000); + restartDataNodes(removeDataNodeWrappers); + LOGGER.info("RemoveDataNodes:{} rejoined successfully.", removeDataNodes); + } catch (Exception e) { + LOGGER.error("RemoveDataNodes rejoin failed."); + Assert.fail(); + } + } + } catch (InconsistentDataException e) { + LOGGER.error("Unexpected error:", e); + } + + try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement())) { + + // Check the data region distribution after removing data nodes + Map> afterRegionMap = getDataRegionMap(statement); + afterRegionMap.forEach( + (key, valueSet) -> { + LOGGER.info("Key: {}, Value: {}", key, valueSet); + if (valueSet.size() != dataReplicateFactor) { + Assert.fail(); + } + }); + + if (rejoinRemovedDataNode) { + ResultSet result = statement.executeQuery(SHOW_DATANODES); + Set allDataNodeId = new HashSet<>(); + while (result.next()) { + allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); + } + Assert.assertEquals(allDataNodeId.size(), dataNodeNum); + } + } catch (InconsistentDataException e) { + LOGGER.error("Unexpected error:", e); + } + } + + private static Set selectRemoveDataNodes( + Set allDataNodeId, Map> regionMap, int removeDataNodeNum) { + Set removeDataNodeIds = new HashSet<>(); + for (int i = 0; i < removeDataNodeNum; i++) { + int removeDataNodeId = allDataNodeId.iterator().next(); + removeDataNodeIds.add(removeDataNodeId); + allDataNodeId.remove(removeDataNodeId); + } + return removeDataNodeIds; + } + + private static void awaitUntilSuccess( + AtomicReference clientRef, + List removeDataNodeLocations) { + AtomicReference> lastTimeDataNodeLocations = new AtomicReference<>(); + AtomicReference lastException = new AtomicReference<>(); + + try { + Awaitility.await() + .atMost(2, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .until( + () -> { + try { + List remainingDataNodes = + clientRef + .get() + .getDataNodeConfiguration(-1) + .getDataNodeConfigurationMap() + .values() + .stream() + .map(TDataNodeConfiguration::getLocation) + .collect(Collectors.toList()); + lastTimeDataNodeLocations.set(remainingDataNodes); + for (TDataNodeLocation location : removeDataNodeLocations) { + if (remainingDataNodes.contains(location)) { + return false; + } + } + return true; + } catch (TException e) { + clientRef.set( + (SyncConfigNodeIServiceClient) + EnvFactory.getEnv().getLeaderConfigNodeConnection()); + lastException.set(e); + return false; + } catch (Exception e) { + // Any exception can be ignored + lastException.set(e); + return false; + } + }); + } catch (ConditionTimeoutException e) { + if (lastTimeDataNodeLocations.get() == null) { + LOGGER.error( + "Maybe getDataNodeConfiguration fail, lastTimeDataNodeLocations is null, last Exception:", + lastException.get()); + throw e; + } + String actualSetStr = lastTimeDataNodeLocations.get().toString(); + lastTimeDataNodeLocations.get().removeAll(removeDataNodeLocations); + String expectedSetStr = lastTimeDataNodeLocations.get().toString(); + LOGGER.error( + "Remove DataNodes timeout in 2 minutes, expected set: {}, actual set: {}", + expectedSetStr, + actualSetStr); + if (lastException.get() == null) { + LOGGER.info("No exception during awaiting"); + } else { + LOGGER.error("Last exception during awaiting:", lastException.get()); + } + throw e; + } + + LOGGER.info("DataNodes has been successfully changed to {}", lastTimeDataNodeLocations.get()); + } + + public void restartDataNodes(List dataNodeWrappers) { + dataNodeWrappers.parallelStream() + .forEach( + nodeWrapper -> { + nodeWrapper.stopForcibly(); + Awaitility.await() + .atMost(1, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .until(() -> !nodeWrapper.isAlive()); + LOGGER.info("Node {} stopped.", nodeWrapper.getId()); + nodeWrapper.start(); + Awaitility.await() + .atMost(1, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .until(nodeWrapper::isAlive); + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + LOGGER.info("Node {} restarted.", nodeWrapper.getId()); + }); + } + + public static String generateRemoveString(Set dataNodes) { + StringBuilder sb = new StringBuilder("remove datanode "); + + for (Integer node : dataNodes) { + sb.append(node).append(", "); + } + + sb.setLength(sb.length() - 2); + + return sb.toString(); + } + + public static Connection getConnectionWithSQLType(SQLModel model) throws SQLException { + if (SQLModel.TABLE_MODEL_SQL.equals(model)) { + return EnvFactory.getEnv().getTableConnection(); + } else { + return EnvFactory.getEnv().getConnection(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java new file mode 100644 index 0000000000000..6dbf5a155ac07 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.removedatanode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@Category({ClusterIT.class}) +@RunWith(IoTDBTestRunner.class) +public class IoTDBRemoveDataNodeNormalIT extends IoTDBRemoveDataNodeITFramework { + + @Test + public void success1C4DTest() throws Exception { + successTest(2, 3, 1, 4, 1, 2, true, SQLModel.NOT_USE_SQL); + } + + @Test + public void fail1C3DTest() throws Exception { + failTest(2, 3, 1, 3, 1, 2, false, SQLModel.NOT_USE_SQL); + } + + @Test + public void success1C4DTestUseSQL() throws Exception { + successTest(2, 3, 1, 4, 1, 2, true, SQLModel.TREE_MODEL_SQL); + } + + @Test + public void fail1C3DTestUseSQL() throws Exception { + failTest(2, 3, 1, 3, 1, 2, false, SQLModel.TREE_MODEL_SQL); + } + + @Test + public void success1C4DTestUseTableSQL() throws Exception { + successTest(2, 3, 1, 4, 1, 2, true, SQLModel.TABLE_MODEL_SQL); + } + + @Test + public void fail1C3DTestUseTableSQL() throws Exception { + failTest(2, 3, 1, 3, 1, 2, false, SQLModel.TABLE_MODEL_SQL); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/SQLModel.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/SQLModel.java new file mode 100644 index 0000000000000..4d6a1c933fa1a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/SQLModel.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.removedatanode; + +public enum SQLModel { + NOT_USE_SQL, + + TREE_MODEL_SQL, + + TABLE_MODEL_SQL, +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java index 76feab5efe32d..5f9da5ac36a83 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java @@ -323,8 +323,8 @@ public static TClusterParameters generateClusterParameters() { clusterParameters.setTimePartitionInterval(604800000); clusterParameters.setDataReplicationFactor(1); clusterParameters.setSchemaReplicationFactor(1); - clusterParameters.setDataRegionPerDataNode(5.0); - clusterParameters.setSchemaRegionPerDataNode(1.0); + clusterParameters.setDataRegionPerDataNode(0); + clusterParameters.setSchemaRegionPerDataNode(1); clusterParameters.setDiskSpaceWarningThreshold(0.01); clusterParameters.setReadConsistencyLevel("strong"); clusterParameters.setTimestampPrecision("ms"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java index bb3a8de7d12f4..620043e6c7b7b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java @@ -128,5 +128,14 @@ public void testBigDateTime() { e.printStackTrace(); fail(); } + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("insert into root.sg.d1(time,s2) values (16182830055860000000, 8.76);"); + fail(); + } catch (SQLException e) { + Assert.assertTrue( + e.getMessage() + .contains("please check whether the timestamp 16182830055860000000 is correct.")); + } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index dd97c18a049ac..576b9c75f2183 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -25,9 +25,9 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.TSStatusCode; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -58,11 +58,11 @@ public class IoTDBDeletionIT { }; private String insertTemplate = - "INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4" + ") VALUES(%d,%d,%d,%f,%s,%b)"; - private String deleteAllTemplate = "DELETE FROM root.vehicle.d0.* WHERE time <= 10000"; + "INSERT INTO root.vehicle%d.d0(timestamp,s0,s1,s2,s3,s4" + ") VALUES(%d,%d,%d,%f,%s,%b)"; + private String deleteAllTemplate = "DELETE FROM root.vehicle%d.d0.* WHERE time <= 10000"; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUp() throws Exception { Locale.setDefault(Locale.ENGLISH); EnvFactory.getEnv() @@ -75,8 +75,8 @@ public void setUp() throws Exception { prepareSeries(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } @@ -90,36 +90,36 @@ public void testUnsupportedValueFilter() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("insert into root.vehicle.d0(time,s0) values (10,310)"); - statement.execute("insert into root.vehicle.d0(time,s3) values (10,'text')"); - statement.execute("insert into root.vehicle.d0(time,s4) values (10,true)"); + statement.execute("insert into root.vehicle1.d0(time,s0) values (10,310)"); + statement.execute("insert into root.vehicle1.d0(time,s3) values (10,'text')"); + statement.execute("insert into root.vehicle1.d0(time,s4) values (10,true)"); String errorMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": For delete statement, where clause can only contain time expressions, value filter is not currently supported."; try { - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE s0 <= 300 AND s0 > 0"); + statement.execute("DELETE FROM root.vehicle1.d0.s0 WHERE s0 <= 300 AND s0 > 0"); fail("should not reach here!"); } catch (SQLException e) { assertEquals(errorMsg, e.getMessage()); } try { - statement.execute("DELETE FROM root.vehicle.d0.s3 WHERE s3 = 'text'"); + statement.execute("DELETE FROM root.vehicle1.d0.s3 WHERE s3 = 'text'"); fail("should not reach here!"); } catch (SQLException e) { assertEquals(errorMsg, e.getMessage()); } try { - statement.execute("DELETE FROM root.vehicle.d0.s4 WHERE s4 != true"); + statement.execute("DELETE FROM root.vehicle1.d0.s4 WHERE s4 != true"); fail("should not reach here!"); } catch (SQLException e) { assertEquals(errorMsg, e.getMessage()); } - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle1.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -127,7 +127,7 @@ public void testUnsupportedValueFilter() throws SQLException { assertEquals(1, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s3 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s3 FROM root.vehicle1.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -135,7 +135,7 @@ public void testUnsupportedValueFilter() throws SQLException { assertEquals(1, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s4 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s4 FROM root.vehicle1.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -147,17 +147,17 @@ public void testUnsupportedValueFilter() throws SQLException { @Test public void test() throws SQLException { - prepareData(); + prepareData(2); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time <= 300"); + statement.execute("DELETE FROM root.vehicle2.d0.s0 WHERE time <= 300"); statement.execute( - "DELETE FROM root.vehicle.d0.s1,root.vehicle.d0.s2,root.vehicle.d0.s3" + "DELETE FROM root.vehicle2.d0.s1,root.vehicle2.d0.s2,root.vehicle2.d0.s3" + " WHERE time <= 350"); - statement.execute("DELETE FROM root.vehicle.d0.** WHERE time <= 150"); + statement.execute("DELETE FROM root.vehicle2.d0.** WHERE time <= 150"); - try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle2.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -165,7 +165,7 @@ public void test() throws SQLException { assertEquals(250, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle2.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -173,7 +173,7 @@ public void test() throws SQLException { assertEquals(100, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s1,s2,s3 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s1,s2,s3 FROM root.vehicle2.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -181,25 +181,25 @@ public void test() throws SQLException { assertEquals(50, cnt); } } - cleanData(); + cleanData(2); } @Test public void testDelAfterFlush() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("CREATE DATABASE root.ln.wf01.wt01"); + statement.execute("CREATE DATABASE root.ln3.wf01.wt01"); statement.execute( - "CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN," + " ENCODING=PLAIN"); + "CREATE TIMESERIES root.ln3.wf01.wt01.status WITH DATATYPE=BOOLEAN," + " ENCODING=PLAIN"); statement.execute( - "INSERT INTO root.ln.wf01.wt01(timestamp,status) " + "values(1509465600000,true)"); - statement.execute("INSERT INTO root.ln.wf01.wt01(timestamp,status) VALUES(NOW(), false)"); + "INSERT INTO root.ln3.wf01.wt01(timestamp,status) " + "values(1509465600000,true)"); + statement.execute("INSERT INTO root.ln3.wf01.wt01(timestamp,status) VALUES(NOW(), false)"); - statement.execute("delete from root.ln.wf01.wt01.status where time <= NOW()"); + statement.execute("delete from root.ln3.wf01.wt01.status where time <= NOW()"); statement.execute("flush"); - statement.execute("delete from root.ln.wf01.wt01.status where time <= NOW()"); + statement.execute("delete from root.ln3.wf01.wt01.status where time <= NOW()"); - try (ResultSet resultSet = statement.executeQuery("select status from root.ln.wf01.wt01")) { + try (ResultSet resultSet = statement.executeQuery("select status from root.ln3.wf01.wt01")) { assertFalse(resultSet.next()); } } @@ -207,13 +207,13 @@ public void testDelAfterFlush() throws SQLException { @Test public void testRangeDelete() throws SQLException { - prepareData(); + prepareData(4); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time <= 300"); - statement.execute("DELETE FROM root.vehicle.d0.s1 WHERE time > 150"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle4.d0.s0 WHERE time <= 300"); + statement.execute("DELETE FROM root.vehicle4.d0.s1 WHERE time > 150"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle4.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -221,7 +221,7 @@ public void testRangeDelete() throws SQLException { assertEquals(100, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s1 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s1 FROM root.vehicle4.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -229,8 +229,8 @@ public void testRangeDelete() throws SQLException { assertEquals(150, cnt); } - statement.execute("DELETE FROM root.vehicle.d0.** WHERE time > 50 and time <= 250"); - try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle4.d0.** WHERE time > 50 and time <= 250"); + try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle4.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -238,22 +238,22 @@ public void testRangeDelete() throws SQLException { assertEquals(200, cnt); } } - cleanData(); + cleanData(4); } @Test public void testFullDeleteWithoutWhereClause() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.s0"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle5.d0.s0"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle5.d0")) { int cnt = 0; while (set.next()) { cnt++; } assertEquals(0, cnt); } - cleanData(); + cleanData(5); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -262,12 +262,12 @@ public void testFullDeleteWithoutWhereClause() { @Test public void testPartialPathRangeDelete() throws SQLException { - prepareData(); + prepareData(6); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.* WHERE time <= 300 and time > 150"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle6.d0.* WHERE time <= 300 and time > 150"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle6.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -275,8 +275,8 @@ public void testPartialPathRangeDelete() throws SQLException { assertEquals(250, cnt); } - statement.execute("DELETE FROM root.vehicle.*.s0 WHERE time <= 100"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle6.*.s0 WHERE time <= 100"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle6.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -284,63 +284,65 @@ public void testPartialPathRangeDelete() throws SQLException { assertEquals(150, cnt); } } - cleanData(); + cleanData(6); } @Test public void testDelFlushingMemtable() throws SQLException { + int testNum = 7; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { for (int i = 1; i <= 10000; i++) { statement.execute( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 1500 and time <= 9000"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle7.d0.s0 WHERE time > 1500 and time <= 9000"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle7.d0")) { int cnt = 0; while (set.next()) { cnt++; } assertEquals(2500, cnt); } - cleanData(); + cleanData(testNum); } } @Test public void testDelMultipleFlushingMemtable() throws SQLException { + int testNum = 8; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { for (int i = 1; i <= 1000; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.executeBatch(); statement.clearBatch(); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 150 and time <= 300"); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 300 and time <= 400"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 150 and time <= 300"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 300 and time <= 400"); for (int i = 1001; i <= 2000; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.executeBatch(); statement.clearBatch(); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 500 and time <= 800"); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 900 and time <= 1100"); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 1500 and time <= 1650"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 500 and time <= 800"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 900 and time <= 1100"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 1500 and time <= 1650"); statement.execute("flush"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle8.d0")) { int cnt = 0; while (set.next()) { cnt++; } assertEquals(1100, cnt); } - cleanData(); + cleanData(testNum); } } @@ -348,10 +350,11 @@ public void testDelMultipleFlushingMemtable() throws SQLException { public void testDeleteAll() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("insert into root.lz.dev.GPS(time, latitude, longitude) values(9,3.2,9.8)"); - statement.execute("insert into root.lz.dev.GPS(time, latitude) values(11,4.5)"); + statement.execute( + "insert into root.lz9.dev.GPS(time, latitude, longitude) values(9,3.2,9.8)"); + statement.execute("insert into root.lz9.dev.GPS(time, latitude) values(11,4.5)"); - try (ResultSet resultSet = statement.executeQuery("select * from root.lz.dev.GPS")) { + try (ResultSet resultSet = statement.executeQuery("select * from root.lz9.dev.GPS")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -359,9 +362,9 @@ public void testDeleteAll() throws SQLException { Assert.assertEquals(2, cnt); } - statement.execute("delete from root.lz.**"); + statement.execute("delete from root.lz9.**"); - try (ResultSet resultSet = statement.executeQuery("select * from root.lz.dev.GPS")) { + try (ResultSet resultSet = statement.executeQuery("select * from root.lz9.dev.GPS")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -376,16 +379,16 @@ public void testDeleteDataFromEmptySeries() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( - "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); + "create timeseries root.`0`.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); statement.execute( - "INSERT INTO root.ln.wf01.wt01(Time,status) VALUES (2022-10-11 10:20:50,true),(2022-10-11 10:20:51,true);"); + "INSERT INTO root.ln10.wf01.wt01(Time,status) VALUES (2022-10-11 10:20:50,true),(2022-10-11 10:20:51,true);"); statement.execute( - "create timeseries root.sg.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); + "create timeseries root.sg10.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); statement.execute( - "DELETE FROM root.ln.wf01.wt01.status,root.sg.wf01.wt01.status WHERE time >2022-10-11 10:20:50;"); + "DELETE FROM root.ln10.wf01.wt01.status,root.sg.wf01.wt01.status WHERE time >2022-10-11 10:20:50;"); - try (ResultSet resultSet = statement.executeQuery("select ** from root")) { + try (ResultSet resultSet = statement.executeQuery("select ** from root.ln10")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -400,11 +403,11 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( - "CREATE TIMESERIES root.ln.d1.`status,01` WITH DATATYPE=BOOLEAN, ENCODING=PLAIN"); - statement.execute("INSERT INTO root.ln.d1(timestamp,`status,01`) VALUES(300, true)"); - statement.execute("INSERT INTO root.ln.d1(timestamp,`status,01`) VALUES(500, false)"); + "CREATE TIMESERIES root.ln11.d1.`status,01` WITH DATATYPE=BOOLEAN, ENCODING=PLAIN"); + statement.execute("INSERT INTO root.ln11.d1(timestamp,`status,01`) VALUES(300, true)"); + statement.execute("INSERT INTO root.ln11.d1(timestamp,`status,01`) VALUES(500, false)"); - try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln.d1")) { + try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln11.d1")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -412,9 +415,9 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { Assert.assertEquals(2, cnt); } - statement.execute("DELETE FROM root.ln.d1.`status,01` WHERE time <= 400"); + statement.execute("DELETE FROM root.ln11.d1.`status,01` WHERE time <= 400"); - try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln.d1")) { + try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln11.d1")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -422,9 +425,9 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { Assert.assertEquals(1, cnt); } - statement.execute("DELETE FROM root.ln.d1.`status,01`"); + statement.execute("DELETE FROM root.ln11.d1.`status,01`"); - try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln.d1")) { + try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln11.d1")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -434,6 +437,36 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { } } + @Test + public void testDelAfterUpdate() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE ALIGNED TIMESERIES root.ln12.d1 (status int32)"); + statement.execute("INSERT INTO root.ln12.d1(timestamp, status) VALUES(1, 1)"); + statement.execute("INSERT INTO root.ln12.d1(timestamp, status) VALUES(2, 2)"); + statement.execute("INSERT INTO root.ln12.d1(timestamp, status) VALUES(3, 3)"); + statement.execute("INSERT INTO root.ln12.d1(timestamp, status) VALUES(2, 2)"); + + try (ResultSet resultSet = statement.executeQuery("select status from root.ln12.d1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(3, cnt); + } + + statement.execute("DELETE FROM root.ln12.d1.* WHERE time <= 2"); + + try (ResultSet resultSet = statement.executeQuery("select status from root.ln12.d1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + } + } + private static void prepareSeries() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -447,40 +480,40 @@ private static void prepareSeries() { } } - private void prepareData() throws SQLException { + private void prepareData(int testNum) throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { // prepare BufferWrite file for (int i = 201; i <= 300; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.addBatch("flush"); // prepare Unseq-File for (int i = 1; i <= 100; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.addBatch("flush"); // prepare BufferWrite cache for (int i = 301; i <= 400; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } // prepare Overflow cache for (int i = 101; i <= 200; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.executeBatch(); } } - private void cleanData() throws SQLException { + private void cleanData(int testNum) throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute(deleteAllTemplate); + statement.execute(String.format(deleteAllTemplate, testNum)); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java index 209657e516185..047bb10bfd3ca 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java @@ -43,8 +43,6 @@ public class IoTDBDuplicateTimeIT { @Before public void setUp() throws Exception { - EnvFactory.getEnv().getConfig().getCommonConfig().setAvgSeriesPointNumberThreshold(2); - // Adjust memstable threshold size to make it flush automatically EnvFactory.getEnv().initClusterEnvironment(); } @@ -62,6 +60,7 @@ public void testDuplicateTime() throws SQLException { // version-1 tsfile statement.execute("insert into root.db.d1(time,s1) values (2,2)"); statement.execute("insert into root.db.d1(time,s1) values (3,3)"); + statement.execute("flush"); // version-2 unseq work memtable statement.execute("insert into root.db.d1(time,s1) values (2,20)"); @@ -69,9 +68,11 @@ public void testDuplicateTime() throws SQLException { // version-3 tsfile statement.execute("insert into root.db.d1(time,s1) values (5,5)"); statement.execute("insert into root.db.d1(time,s1) values (6,6)"); + statement.execute("flush root.db true"); // version-2 unseq work memtable -> unseq tsfile statement.execute("insert into root.db.d1(time,s1) values (5,50)"); + statement.execute("flush"); try (ResultSet set = statement.executeQuery("SELECT s1 FROM root.db.d1 where time = 5")) { int cnt = 0; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java index aa7b7024677c0..6bcbc762fc53b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java @@ -19,13 +19,17 @@ package org.apache.iotdb.db.it; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -36,22 +40,37 @@ import java.sql.Statement; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class}) public class IoTDBEncodingIT { - @Before - public void setUp() throws Exception { + private static final String[] databasesToClear = + new String[] {"root.db_0", "root.db1", "root.turbine1"}; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @After + public void tearDown() { + for (String database : databasesToClear) { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("DELETE DATABASE " + database); + } catch (Exception ignored) { + + } + } + } + @Test public void testSetEncodingRegularFailed() { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -432,4 +451,75 @@ public void testFloatPrecision2() { fail(); } } + + @Test + public void testCreateNewTypes() throws Exception { + String currDB = "root.db1"; + int seriesCnt = 0; + TSDataType[] dataTypes = + new TSDataType[] { + TSDataType.STRING, TSDataType.BLOB, TSDataType.TIMESTAMP, TSDataType.DATE + }; + + // supported encodings + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (TSDataType dataType : dataTypes) { + for (TSEncoding encoding : TSEncoding.values()) { + if (encoding.isSupported(dataType)) { + statement.execute( + "create timeseries " + + currDB + + ".d1.s" + + seriesCnt + + " with datatype=" + + dataType + + ", encoding=" + + encoding + + ", compression=SNAPPY"); + seriesCnt++; + } + } + } + + ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES"); + + while (resultSet.next()) { + seriesCnt--; + } + assertEquals(0, seriesCnt); + statement.execute("DROP DATABASE " + currDB); + } + + // unsupported encodings + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (TSDataType dataType : dataTypes) { + for (TSEncoding encoding : TSEncoding.values()) { + if (!encoding.isSupported(dataType)) { + try { + statement.execute( + "create timeseries " + + currDB + + ".d1.s" + + seriesCnt + + " with datatype=" + + dataType + + ", encoding=" + + encoding + + ", compression=SNAPPY"); + fail("Should have thrown an exception"); + } catch (SQLException e) { + assertEquals( + "507: encoding " + encoding + " does not support " + dataType, e.getMessage()); + } + seriesCnt++; + } + } + } + + ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES"); + assertFalse(resultSet.next()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExampleIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExampleIT.java index 424195bdd37ba..be8ddabeb2cd6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExampleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExampleIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java index f1822fd4bb420..61ba53f8bd475 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFileTimeIndexIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFileTimeIndexIT.java new file mode 100644 index 0000000000000..a0d0a083701ac --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFileTimeIndexIT.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBFileTimeIndexIT { + + private static final String[] sqls = + new String[] { + "insert into root.db.d1(time,s1) values(2,2)", + "insert into root.db.d1(time,s1) values(3,3)", + "flush", + "insert into root.db.d2(time,s1) values(5,5)", + "flush", + "insert into root.db.d1(time,s1) values(4,4)", + "flush", + "insert into root.db.d2(time,s1) values(1,1)", + "insert into root.db.d1(time,s1) values(3,30)", + "insert into root.db.d1(time,s1) values(4,40)", + "flush", + "insert into root.db.d2(time,s1) values(2,2)", + "insert into root.db.d1(time,s1) values(4,400)", + "flush", + }; + + @BeforeClass + public static void setUp() throws Exception { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1) + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setQueryMemoryProportion("1:100:200:50:200:200:0:250"); + // Adjust memstable threshold size to make it flush automatically + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + private static void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testQuery() throws SQLException { + long[] time = {2L, 3L, 4L}; + double[] value = {2.0f, 30.0f, 400.0f}; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select s1 from root.db.d1")) { + int cnt = 0; + while (resultSet.next()) { + assertEquals(time[cnt], resultSet.getLong(1)); + assertEquals(value[cnt], resultSet.getDouble(2), 0.00001); + cnt++; + } + assertEquals(time.length, cnt); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterBetweenIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterBetweenIT.java index 2ce213eb0b023..0ecd4d88f903d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterBetweenIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterBetweenIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java index 705f4b8d9ede7..a7acbaebaa964 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java index c72c7e41b1253..3a73be3adb309 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java @@ -169,4 +169,49 @@ public void selectAllSQLTest() { fail(e.getMessage()); } } + + @Test + public void bigFloatNumberTest2() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + float[] floats = new float[] {6.5536403E8F, 3.123456768E20F, Float.NaN}; + double[] doubles = new double[] {9.223372036854E18, 9.223372036854E100, Double.NaN}; + + statement.execute("create timeseries root.sg.d1.s1 with datatype=float, encoding=rle"); + statement.execute("create timeseries root.sg.d1.s2 with datatype=double, encoding=rle"); + statement.execute( + "insert into root.sg.d1(time, s1, s2) values (1, 6.5536403E8, 9.223372036854E18)"); + statement.execute( + "insert into root.sg.d1(time, s1, s2) values (2, 3.123456768E20, 9.223372036854E100)"); + statement.execute("insert into root.sg.d1(time, s1, s2) values (3, NaN, NaN)"); + + int cnt; + try (ResultSet resultSet = statement.executeQuery("select s1, s2 from root.sg.d1")) { + assertNotNull(resultSet); + cnt = 0; + while (resultSet.next()) { + assertEquals(floats[cnt], resultSet.getFloat("root.sg.d1.s1"), DELTA_FLOAT); + assertEquals(doubles[cnt], resultSet.getDouble("root.sg.d1.s2"), DELTA_DOUBLE); + cnt++; + } + assertEquals(3, cnt); + } + + statement.execute("flush"); + + try (ResultSet resultSet = statement.executeQuery("select s1, s2 from root.sg.d1")) { + assertNotNull(resultSet); + cnt = 0; + while (resultSet.next()) { + assertEquals(floats[cnt], resultSet.getFloat("root.sg.d1.s1"), DELTA_FLOAT); + assertEquals(doubles[cnt], resultSet.getDouble("root.sg.d1.s2"), DELTA_DOUBLE); + cnt++; + } + assertEquals(3, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java index 7ebbcfc8472b3..18c00b125dd1c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java @@ -56,7 +56,7 @@ public class IoTDBInsertMultiRowIT { @BeforeClass public static void setUp() throws Exception { - EnvFactory.getEnv().getConfig().getCommonConfig().setMaxInnerCompactionCandidateFileNum(2); + EnvFactory.getEnv().getConfig().getCommonConfig().setInnerCompactionCandidateFileNum(2); EnvFactory.getEnv().initClusterEnvironment(); initCreateSQLStatement(); insertData(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertNaNIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertNaNIT.java index 7d37908d6098c..a0ffd6e383f5d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertNaNIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertNaNIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java index d237423c52f41..a5032580f2250 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java @@ -52,6 +52,7 @@ public class IoTDBInsertWithQueryIT { @Before public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false); EnvFactory.getEnv().initClusterEnvironment(); } @@ -450,9 +451,9 @@ private void queryNegativeTimestampTypeDataTest() { }; String[] retArray = new String[] { - "-9223372036854775807,-9223372036854775807,", - "-2208952800000,-2208952800000,", - "-999999,-9999999," + "-9223372036854775807,-292275055-05-16T16:47:04.193Z,", + "-2208952800000,1900-01-01T10:00:00.000Z,", + "-999999,1969-12-31T21:13:20.001Z," }; resultSetEqualTest("SELECT s2 FROM root.fans.d0;", expectedHeader, retArray); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index c1b1fdf91091f..94b1941ec56f2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -20,17 +20,22 @@ package org.apache.iotdb.db.it; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.it.utils.TsFileGenerator; +import org.apache.iotdb.it.utils.TsFileTableGenerator; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.jdbc.IoTDBSQLException; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -47,6 +52,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -55,6 +61,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.createUser; @@ -133,17 +140,17 @@ private String convert2SQL(final String device, final MeasurementSchema schema) final String sql = String.format( "create timeseries %s %s", - new Path(device, schema.getMeasurementId(), true).getFullPath(), + new Path(device, schema.getMeasurementName(), true).getFullPath(), schema.getType().name()); LOGGER.info("schema execute: {}", sql); return sql; } - private String convert2AlignedSQL(final String device, final List schemas) { + private String convert2AlignedSQL(final String device, final List schemas) { String sql = String.format("create aligned timeseries %s(", device); for (int i = 0; i < schemas.size(); i++) { - final MeasurementSchema schema = schemas.get(i); - sql += (String.format("%s %s", schema.getMeasurementId(), schema.getType().name())); + final IMeasurementSchema schema = schemas.get(i); + sql += (String.format("%s %s", schema.getMeasurementName(), schema.getType().name())); sql += (i == schemas.size() - 1 ? ")" : ","); } LOGGER.info("schema execute: {}.", sql); @@ -244,7 +251,7 @@ public void testLoad() throws Exception { statement.execute( String.format( "delete timeseries %s.%s", - SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00.getMeasurementId())); + SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00.getMeasurementName())); } } @@ -305,14 +312,14 @@ public void testLoadWithExtendTemplate() throws Exception { Arrays.asList( "lat", "lon", - SchemaConfig.MEASUREMENT_00.getMeasurementId(), - SchemaConfig.MEASUREMENT_01.getMeasurementId(), - SchemaConfig.MEASUREMENT_02.getMeasurementId(), - SchemaConfig.MEASUREMENT_03.getMeasurementId(), - SchemaConfig.MEASUREMENT_04.getMeasurementId(), - SchemaConfig.MEASUREMENT_05.getMeasurementId(), - SchemaConfig.MEASUREMENT_06.getMeasurementId(), - SchemaConfig.MEASUREMENT_07.getMeasurementId())); + SchemaConfig.MEASUREMENT_00.getMeasurementName(), + SchemaConfig.MEASUREMENT_01.getMeasurementName(), + SchemaConfig.MEASUREMENT_02.getMeasurementName(), + SchemaConfig.MEASUREMENT_03.getMeasurementName(), + SchemaConfig.MEASUREMENT_04.getMeasurementName(), + SchemaConfig.MEASUREMENT_05.getMeasurementName(), + SchemaConfig.MEASUREMENT_06.getMeasurementName(), + SchemaConfig.MEASUREMENT_07.getMeasurementName())); try (final ResultSet resultSet = statement.executeQuery("show nodes in schema template t1")) { while (resultSet.next()) { String device = resultSet.getString(ColumnHeaderConstant.CHILD_NODES); @@ -327,7 +334,7 @@ public void testLoadWithExtendTemplate() throws Exception { } @Test - public void testLoadWithAutoRegister() throws Exception { + public void testLoadWithAutoCreate() throws Exception { final long writtenPoint1; // device 0, device 1, sg 0 try (final TsFileGenerator generator = @@ -542,7 +549,9 @@ public void testLoadWithOnSuccess() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( - String.format("load \"%s\" sglevel=2 onSuccess=none", file1.getAbsolutePath())); + String.format( + "load \"%s\" with ('database-level'='2', 'on-success'='none')", + file1.getAbsolutePath())); try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { @@ -560,7 +569,9 @@ public void testLoadWithOnSuccess() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( - String.format("load \"%s\" sglevel=2 onSuccess=delete", file2.getAbsolutePath())); + String.format( + "load \"%s\" with ('database-level'='2', 'on-success'='delete')", + file2.getAbsolutePath())); try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { @@ -582,7 +593,7 @@ public void testLoadWithLastCache() throws Exception { registerSchema(); final String device = SchemaConfig.DEVICE_0; - final String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementId(); + final String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementName(); try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { @@ -760,7 +771,6 @@ public void testLoadWithMods() throws Exception { generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true); generator.generateDeletion(SchemaConfig.DEVICE_0, 10); - generator.generateDeletion(SchemaConfig.DEVICE_1, 10); writtenPoint1 = generator.getTotalNumber(); } @@ -894,7 +904,202 @@ public void testLoadLocally() throws Exception { } } + @Test + public void testLoadWithConvertOnTypeMismatchForTreeModel() throws Exception { + + List> measurementSchemas = + generateMeasurementSchemasForDataTypeConvertion(); + + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + long writtenPoint = 0; + List schemaList1 = + measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + List schemaList2 = + measurementSchemas.stream().map(pair -> pair.right).collect(Collectors.toList()); + + try (final TsFileGenerator generator = new TsFileGenerator(file)) { + generator.registerTimeseries(SchemaConfig.DEVICE_0, schemaList2); + + generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); + + writtenPoint = generator.getTotalNumber(); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + for (MeasurementSchema schema : schemaList1) { + statement.execute(convert2SQL(SchemaConfig.DEVICE_0, schema)); + } + + statement.execute(String.format("load \"%s\" ", file.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(*) from root.** group by level=1,2")) { + if (resultSet.next()) { + final long sgCount = resultSet.getLong("count(root.sg.test_0.*.*)"); + Assert.assertEquals(writtenPoint, sgCount); + } else { + Assert.fail("This ResultSet is empty."); + } + } + } + } + + private List> + generateMeasurementSchemasForDataTypeConvertion() { + TSDataType[] dataTypes = { + TSDataType.STRING, + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.TIMESTAMP, + TSDataType.BOOLEAN, + TSDataType.DATE, + TSDataType.DOUBLE, + TSDataType.FLOAT, + TSDataType.INT32, + TSDataType.INT64 + }; + List> pairs = new ArrayList<>(); + + for (TSDataType type : dataTypes) { + for (TSDataType dataType : dataTypes) { + String id = String.format("%s2%s", type.name(), dataType.name()); + pairs.add(new Pair<>(new MeasurementSchema(id, type), new MeasurementSchema(id, dataType))); + } + } + return pairs; + } + + @Test + public void testLoadWithEmptyDatabaseForTableModel() throws Exception { + final int lineCount = 10000; + + final List> measurementSchemas = + generateMeasurementSchemasForDataTypeConvertion(); + final List columnCategories = + generateTabletColumnCategory(0, measurementSchemas.size()); + + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + final List schemaList = + measurementSchemas.stream().map(pair -> pair.right).collect(Collectors.toList()); + + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, schemaList, columnCategories); + + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000); + } + + // Prepare normal user + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + adminStmt.execute( + String.format( + "grant create, insert on %s.%s to user test", + SchemaConfig.DATABASE_0, SchemaConfig.TABLE_0)); + + // auto-create table + adminStmt.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("load '%s'", file.getAbsolutePath())); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + try (final ResultSet resultSet = + adminStmt.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { + if (resultSet.next()) { + Assert.assertEquals(lineCount, resultSet.getLong(1)); + } else { + Assert.fail("This ResultSet is empty."); + } + } + } + } + + @Test + public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception { + final int lineCount = 10000; + + List> measurementSchemas = + generateMeasurementSchemasForDataTypeConvertion(); + List columnCategories = + generateTabletColumnCategory(0, measurementSchemas.size()); + + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + List schemaList1 = + measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + List schemaList2 = + measurementSchemas.stream().map(pair -> pair.right).collect(Collectors.toList()); + + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, schemaList2, columnCategories); + + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(convert2TableSQL(SchemaConfig.TABLE_0, schemaList1, columnCategories)); + statement.execute(String.format("load '%s'", file.getAbsolutePath())); + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { + if (resultSet.next()) { + Assert.assertEquals(lineCount, resultSet.getLong(1)); + } else { + Assert.fail("This ResultSet is empty."); + } + } + } + } + + private List generateTabletColumnCategory(int tagNum, int filedNum) { + List columnTypes = new ArrayList<>(tagNum + filedNum); + for (int i = 0; i < tagNum; i++) { + columnTypes.add(ColumnCategory.TAG); + } + for (int i = 0; i < filedNum; i++) { + columnTypes.add(ColumnCategory.FIELD); + } + return columnTypes; + } + + private String convert2TableSQL( + final String tableName, + final List schemaList, + final List columnCategoryList) { + List columns = new ArrayList<>(); + for (int i = 0; i < schemaList.size(); i++) { + final MeasurementSchema measurement = schemaList.get(i); + columns.add( + String.format( + "%s %s %s", + measurement.getMeasurementName(), + measurement.getType(), + columnCategoryList.get(i).name())); + } + String tableCreation = + String.format("create table %s(%s)", tableName, String.join(", ", columns)); + LOGGER.info("schema execute: {}", tableCreation); + return tableCreation; + } + private static class SchemaConfig { + private static final String DATABASE_0 = "root"; + private static final String TABLE_0 = "test"; private static final String STORAGE_GROUP_0 = "root.sg.test_0"; private static final String STORAGE_GROUP_1 = "root.sg.test_1"; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java new file mode 100644 index 0000000000000..9d0a54f16728c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.exception.DataRegionException; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.SealedTsFileRecoverPerformer; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.Objects; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBLoadTsFileWithModIT { + private File tmpDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(Files.createTempDirectory("load").toUri()); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + try { + for (final File file : Objects.requireNonNull(tmpDir.listFiles())) { + Files.delete(file.toPath()); + } + Files.delete(tmpDir.toPath()); + } finally { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + } + + private void generateFileWithNewModFile() + throws IOException, WriteProcessException, IllegalPathException, DataRegionException { + TsFileResource resource = generateFile(); + // write mods file + resource + .getExclusiveModFile() + .write(new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1"), 1, 2)); + resource.getExclusiveModFile().close(); + } + + private void generateFileWithOldModFile() + throws IOException, DataRegionException, WriteProcessException, IllegalPathException { + TsFileResource resource = generateFile(); + ModificationFileV1 oldModFile = ModificationFileV1.getNormalMods(resource); + oldModFile.write(new Deletion(new MeasurementPath("root.test.d1.s1"), Long.MAX_VALUE, 1, 2)); + oldModFile.close(); + } + + private TsFileResource generateFile() + throws WriteProcessException, IOException, DataRegionException { + File tsfile = new File(tmpDir, "1-1-0-0.tsfile"); + try (TsFileWriter writer = new TsFileWriter(tsfile)) { + writer.registerAlignedTimeseries( + "root.test.d1", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + Tablet tablet = + new Tablet( + "root.test.d1", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + for (int i = 0; i < 5; i++) { + tablet.addTimestamp(i, i); + tablet.addValue(i, 0, true); + } + writer.writeTree(tablet); + } + // generate resource file + TsFileResource resource = new TsFileResource(tsfile); + try (SealedTsFileRecoverPerformer performer = new SealedTsFileRecoverPerformer(resource)) { + performer.recover(); + } + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + resource.deserialize(); + return resource; + } + + @Test + public void testWithNewModFile() + throws SQLException, + IOException, + DataRegionException, + WriteProcessException, + IllegalPathException { + generateFileWithNewModFile(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \'%s\'", tmpDir.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(s1) as c from root.test.d1")) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(3, resultSet.getLong("c")); + } + } + } + + @Test + public void testWithOldModFile() + throws SQLException, + IOException, + DataRegionException, + WriteProcessException, + IllegalPathException { + generateFileWithOldModFile(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \'%s\'", tmpDir.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(s1) as c from root.test.d1")) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(3, resultSet.getLong("c")); + Assert.assertTrue( + new File(tmpDir, "1-1-0-0.tsfile" + ModificationFileV1.FILE_SUFFIX).exists()); + Assert.assertFalse( + new File(tmpDir, "1-1-0-0.tsfile" + ModificationFile.FILE_SUFFIX).exists()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java index 205f8b0091069..ab289ef6affcf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -624,12 +625,8 @@ public void testRegularLikeInExpressions() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { String query = - "SELECT s1 FROM root.vehicle.d1 WHERE s3 LIKE '_' && s3 REGEXP '[0-9]' && s3 IN ('4', '2', '3')"; + "SELECT s1 FROM root.vehicle.d1 WHERE s3 LIKE '_' && s3 not REGEXP '[0-9]' && s3 IN ('4', '2', '3')"; try (ResultSet rs = statement.executeQuery(query)) { - for (int i = 2; i <= 4; i++) { - Assert.assertTrue(rs.next()); - Assert.assertEquals(i, rs.getLong(1)); - } Assert.assertFalse(rs.next()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java index 074da6873eea0..18f805d1354fd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -108,7 +109,7 @@ public void testPartialInsertionRestart() throws SQLException { EnvironmentUtils.restartDaemon(); StorageEngine.getInstance().recover(); // wait for recover - while (!StorageEngine.getInstance().isAllSgReady()) { + while (!StorageEngine.getInstance().isReadyForReadAndWrite()) { Thread.sleep(500); time += 500; if (time > 10000) { @@ -146,28 +147,28 @@ public void testPartialInsertTablet() { "root.sg1.d1.s1", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( "root.sg1.d1.s2", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); Tablet tablet = new Tablet("root.sg1.d1", schemaList, 300); long timestamp = 0; for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = timestamp; - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } timestamp++; } timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = timestamp; - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } timestamp++; } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java index 41bcc24a375a3..757b514ff13ef 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java @@ -52,7 +52,6 @@ public class IoTDBRecoverIT { private static final Logger logger = LoggerFactory.getLogger(IoTDBRecoverIT.class); - private static final String TIMESTAMP_STR = "Time"; private static final String TEMPERATURE_STR = "root.ln.wf01.wt01.temperature"; private static final String[] creationSqls = new String[] { @@ -110,7 +109,7 @@ public void recoverTest1() { logger.info("All DataNodes are started"); // check cluster whether restart ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); - String[] retArray = new String[] {"0,2", "0,4", "0,3"}; + String[] retArray = new String[] {"2", "4", "3"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -118,8 +117,7 @@ public void recoverTest1() { try (ResultSet resultSet = statement.executeQuery(selectSql)) { assertNotNull(resultSet); resultSet.next(); - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(TEMPERATURE_STR)); + String ans = resultSet.getString(count(TEMPERATURE_STR)); Assert.assertEquals(retArray[0], ans); } @@ -127,10 +125,7 @@ public void recoverTest1() { try (ResultSet resultSet = statement.executeQuery(selectSql)) { assertNotNull(resultSet); resultSet.next(); - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(TEMPERATURE_STR)); + String ans = resultSet.getString(minTime(TEMPERATURE_STR)); Assert.assertEquals(retArray[1], ans); } @@ -138,10 +133,7 @@ public void recoverTest1() { try (ResultSet resultSet = statement.executeQuery(selectSql)) { assertNotNull(resultSet); resultSet.next(); - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(TEMPERATURE_STR)); + String ans = resultSet.getString(minTime(TEMPERATURE_STR)); Assert.assertEquals(retArray[2], ans); } @@ -151,7 +143,7 @@ public void recoverTest1() { } // max min ValueTest - retArray = new String[] {"0,8499,500.0", "0,2499,500.0"}; + retArray = new String[] {"8499,500.0", "2499,500.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -162,11 +154,7 @@ public void recoverTest1() { assertNotNull(resultSet); resultSet.next(); String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) - + "," - + resultSet.getString(minValue(d0s2)); + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(minValue(d0s2)); Assert.assertEquals(retArray[0], ans); } @@ -175,11 +163,7 @@ public void recoverTest1() { assertNotNull(resultSet); resultSet.next(); String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) - + "," - + resultSet.getString(minValue(d0s2)); + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(minValue(d0s2)); Assert.assertEquals(retArray[1], ans); } } catch (Exception e) { @@ -202,7 +186,7 @@ public void recoverTest2() { // wait for cluster to start and check ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); // count test - String[] retArray = new String[] {"0,2001,2001,2001,2001", "0,7500,7500,7500,7500"}; + String[] retArray = new String[] {"2001,2001,2001,2001", "7500,7500,7500,7500"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -213,9 +197,7 @@ public void recoverTest2() { assertNotNull(resultSet); resultSet.next(); String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -230,9 +212,7 @@ public void recoverTest2() { assertNotNull(resultSet); resultSet.next(); String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java index 01a3c79187653..6ea26e7bc7008 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java @@ -96,7 +96,7 @@ public void tearDown() throws Exception { @Test public void test() throws SQLException, IOException { - String[] retArray = new String[] {"0,2", "0,4", "0,3"}; + String[] retArray = new String[] {"2", "4", "3"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -104,8 +104,7 @@ public void test() throws SQLException, IOException { try (ResultSet resultSet = statement.executeQuery(selectSql)) { assertNotNull(resultSet); resultSet.next(); - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(TEMPERATURE_STR)); + String ans = resultSet.getString(count(TEMPERATURE_STR)); assertEquals(retArray[0], ans); } @@ -113,10 +112,7 @@ public void test() throws SQLException, IOException { try (ResultSet resultSet = statement.executeQuery(selectSql)) { assertNotNull(resultSet); resultSet.next(); - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(TEMPERATURE_STR)); + String ans = resultSet.getString(minTime(TEMPERATURE_STR)); assertEquals(retArray[1], ans); } @@ -124,10 +120,7 @@ public void test() throws SQLException, IOException { try (ResultSet resultSet = statement.executeQuery(selectSql)) { assertNotNull(resultSet); resultSet.next(); - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(TEMPERATURE_STR)); + String ans = resultSet.getString(minTime(TEMPERATURE_STR)); assertEquals(retArray[2], ans); } @@ -163,7 +156,7 @@ public void test() throws SQLException, IOException { assertEquals(7500, tempResultSet.getInt("count(" + d0s0 + ")")); // test max, min value - retArray = new String[] {"0,8499,500.0", "0,2499,500.0"}; + retArray = new String[] {"8499,500.0", "2499,500.0"}; selectSql = "select max_value(s0),min_value(s2) " + "from root.vehicle.d0 where time >= 100 and time < 9000"; @@ -171,11 +164,7 @@ public void test() throws SQLException, IOException { assertNotNull(resultSet); resultSet.next(); String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) - + "," - + resultSet.getString(minValue(d0s2)); + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(minValue(d0s2)); assertEquals(retArray[0], ans); } @@ -184,11 +173,7 @@ public void test() throws SQLException, IOException { assertNotNull(resultSet); resultSet.next(); String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s1)) - + "," - + resultSet.getString(minValue(d0s2)); + resultSet.getString(maxValue(d0s1)) + "," + resultSet.getString(minValue(d0s2)); assertEquals(retArray[1], ans); } } catch (Exception e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java index 69528cf865b16..15ad2983ca87c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java @@ -25,7 +25,7 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.PlainDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -98,7 +98,7 @@ private File generateUnsortedFile() throws IOException { Files.createFile(tsfile.toPath()); try (TsFileIOWriter writer = new TsFileIOWriter(tsfile)) { - writer.startChunkGroup(new PlainDeviceID("root.testsg.d1")); + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1")); ChunkWriterImpl chunkWriter = new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT32)); chunkWriter.write(2, 1); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java index 53b92918877d5..7cde457761c4f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -53,7 +53,7 @@ import java.util.List; import java.util.Map; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.COLUMN_TTL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.COLUMN_TTL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1115,8 +1115,8 @@ public void showFunctions(CloseableHttpClient httpClient) { Map map = queryMetaData(httpClient, sql); List columnNamesResult = (List) map.get("columnNames"); List> valuesResult = (List>) map.get("values"); - assertEquals(3, columnNamesResult.size()); - assertEquals(3, valuesResult.size()); + assertEquals(4, columnNamesResult.size()); + assertEquals(4, valuesResult.size()); } public void showTimeseries(CloseableHttpClient httpClient) { @@ -1767,8 +1767,8 @@ public void showFunctionsV2(CloseableHttpClient httpClient) { Map map = queryMetaDataV2(httpClient, sql); List columnNamesResult = (List) map.get("column_names"); List> valuesResult = (List>) map.get("values"); - assertEquals(3, columnNamesResult.size()); - assertEquals(3, valuesResult.size()); + assertEquals(4, columnNamesResult.size()); + assertEquals(4, valuesResult.size()); } public void showTimeseriesV2(CloseableHttpClient httpClient) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java index 069036e10a534..14d298e31bdb4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java @@ -16,10 +16,12 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -27,7 +29,6 @@ import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -45,7 +46,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; -@Ignore @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBRestartIT { @@ -54,12 +54,22 @@ public class IoTDBRestartIT { @Before public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.ratis.RatisConsensus"); EnvFactory.getEnv().initClusterEnvironment(); } @After public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("ASYNC"); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus"); } @Test @@ -71,8 +81,7 @@ public void testRestart() throws SQLException { } try { - // TODO: replace restartDaemon() with new methods in Env. - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); } catch (Exception e) { fail(e.getMessage()); } @@ -83,7 +92,7 @@ public void testRestart() throws SQLException { } try { - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); } catch (Exception e) { fail(e.getMessage()); } @@ -114,23 +123,7 @@ public void testRestartDelete() throws SQLException { statement.execute("insert into root.turbine.d1(timestamp,s1) values(3,3)"); } - long time = 0; - /* - try { - EnvironmentUtils.restartDaemon(); - StorageEngine.getInstance().recover(); - // wait for recover - while (!StorageEngine.getInstance().isAllSgReady()) { - Thread.sleep(500); - time += 500; - if (time > 10000) { - logger.warn("wait too long in restart, wait for: " + time / 1000 + "s"); - } - } - } catch (Exception e) { - fail(e.getMessage()); - } - */ + TestUtils.restartDataNodes(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -174,7 +167,7 @@ public void testRestartQueryLargerThanEndTime() throws SQLException { } try { - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); } catch (Exception e) { fail(e.getMessage()); } @@ -186,7 +179,7 @@ public void testRestartQueryLargerThanEndTime() throws SQLException { } try { - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); } catch (Exception e) { fail(e.getMessage()); } @@ -221,7 +214,7 @@ public void testRestartEndTime() throws SQLException { } try { - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); } catch (Exception e) { fail(e.getMessage()); } @@ -233,7 +226,7 @@ public void testRestartEndTime() throws SQLException { } try { - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); } catch (Exception e) { fail(e.getMessage()); } @@ -265,7 +258,7 @@ public void testRecoverWALMismatchDataType() throws Exception { "create timeseries root.turbine1.d1.s1 with datatype=INT32, encoding=RLE, compression=SNAPPY"); } - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -293,7 +286,7 @@ public void testRecoverWALDeleteSchema() throws Exception { statement.execute("delete timeseries root.turbine1.d1.s1"); } - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -315,8 +308,6 @@ public void testRecoverWALDeleteSchema() throws Exception { @Test public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - int avgSeriesPointNumberThreshold = config.getAvgSeriesPointNumberThreshold(); - config.setAvgSeriesPointNumberThreshold(2); long tsFileSize = config.getSeqTsFileSize(); long unFsFileSize = config.getSeqTsFileSize(); config.setSeqTsFileSize(10000000); @@ -327,13 +318,13 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { statement.execute("create timeseries root.turbine1.d1.s1 with datatype=INT64"); statement.execute("insert into root.turbine1.d1(timestamp,s1) values(1,1)"); statement.execute("insert into root.turbine1.d1(timestamp,s1) values(2,1)"); + statement.execute("flush"); statement.execute("create timeseries root.turbine1.d1.s2 with datatype=BOOLEAN"); statement.execute("insert into root.turbine1.d1(timestamp,s2) values(3,true)"); statement.execute("insert into root.turbine1.d1(timestamp,s2) values(4,true)"); } - Thread.sleep(1000); - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -350,7 +341,6 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { assertEquals(2, cnt); } - config.setAvgSeriesPointNumberThreshold(avgSeriesPointNumberThreshold); config.setSeqTsFileSize(tsFileSize); config.setUnSeqTsFileSize(unFsFileSize); } @@ -363,7 +353,7 @@ public void testRecoverFromFlushMemTableError() throws Exception { } // mock exception during flush memtable - // EnvironmentUtils.restartDaemon(); + TestUtils.restartDataNodes(); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java index f6e9f98726b5b..4df8f79af15ae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java @@ -38,10 +38,15 @@ import java.nio.file.Files; import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class}) public class IoTDBSetConfigurationIT { @@ -101,7 +106,8 @@ public void testSetClusterName() throws Exception { EnvFactory.getEnv().getDataNodeWrapper(0).start(); // set cluster name on datanode Awaitility.await() - .atMost(10, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) + .pollDelay(1, TimeUnit.SECONDS) .until( () -> { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -120,6 +126,9 @@ public void testSetClusterName() throws Exception { .until(() -> !EnvFactory.getEnv().getDataNodeWrapper(0).isAlive()); Assert.assertTrue( checkConfigFileContains(EnvFactory.getEnv().getDataNodeWrapper(0), "cluster_name=yy")); + + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv().initClusterEnvironment(); } private static boolean checkConfigFileContains( @@ -138,4 +147,56 @@ private static boolean checkConfigFileContains( return false; } } + + @Test + public void testSetDefaultSGLevel() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // legal value + statement.execute("set configuration \"default_storage_group_level\"=\"3\""); + statement.execute("INSERT INTO root.a.b.c.d1(timestamp, s1) VALUES (1, 1)"); + ResultSet databases = statement.executeQuery("show databases"); + databases.next(); + Assert.assertEquals("root.a.b.c", databases.getString(1)); + assertFalse(databases.next()); + + // path too short + try { + statement.execute("INSERT INTO root.fail(timestamp, s1) VALUES (1, 1)"); + } catch (SQLException e) { + assertEquals( + "509: root.fail is not a legal path, because it is no longer than default sg level: 3", + e.getMessage()); + } + + // illegal value + try { + statement.execute("set configuration \"default_storage_group_level\"=\"-1\""); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Illegal defaultStorageGroupLevel: -1, should >= 1")); + } + + // Failed updates will not change the files. + assertFalse( + checkConfigFileContains( + EnvFactory.getEnv().getDataNodeWrapper(0), "default_storage_group_level=-1")); + assertTrue( + checkConfigFileContains( + EnvFactory.getEnv().getDataNodeWrapper(0), "default_storage_group_level=3")); + } + + // can start with an illegal value + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultStorageGroupLevel(-1); + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO root.a.b.c.d1(timestamp, s1) VALUES (1, 1)"); + ResultSet databases = statement.executeQuery("show databases"); + databases.next(); + // the default value should take effect + Assert.assertEquals("root.a", databases.getString(1)); + assertFalse(databases.next()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index 19b487193834b..dfc6a83cbbce0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -1041,25 +1041,22 @@ public void testInvalidMaxPointNumber() { @Test public void testStorageGroupWithHyphenInName() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.setFetchSize(5); statement.execute("CREATE DATABASE root.group_with_hyphen"); - } catch (SQLException e) { + } catch (final SQLException e) { fail(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { while (resultSet.next()) { - StringBuilder builder = new StringBuilder(); - builder.append(resultSet.getString(1)); - Assert.assertEquals(builder.toString(), "root.group_with_hyphen"); + Assert.assertEquals("root.group_with_hyphen", resultSet.getString(1)); } } - } catch (SQLException e) { + } catch (final SQLException e) { fail(); } } @@ -1188,4 +1185,42 @@ public void testNewDataType() { fail(); } } + + @Test + public void testIllegalDateType() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute("CREATE DATABASE root.sg1"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s4 WITH DATATYPE=DATE, ENCODING=PLAIN, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s5 WITH DATATYPE=TIMESTAMP, ENCODING=PLAIN, COMPRESSOR=SNAPPY"); + try { + statement.execute("insert into root.sg1.d1(timestamp, s4) values(1, '2022-04-31')"); + fail(); + } catch (Exception e) { + assertEquals( + TSStatusCode.METADATA_ERROR.getStatusCode() + + ": Fail to insert measurements [s4] caused by [data type is not consistent, " + + "input '2022-04-31', registered DATE because Invalid date format. " + + "Please use YYYY-MM-DD format.]", + e.getMessage()); + } + try { + statement.execute( + "insert into root.sg1.d1(timestamp, s5) values(1999-04-31T00:00:00.000+08:00, 1999-04-31T00:00:00.000+08:00)"); + fail(); + } catch (Exception e) { + assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Input time format 1999-04-31T00:00:00.000+08:00 error. " + + "Input like yyyy-MM-dd HH:mm:ss, yyyy-MM-ddTHH:mm:ss " + + "or refer to user document for more info.", + e.getMessage()); + } + } catch (SQLException e) { + fail(); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionIdentifierIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionIdentifierIT.java index 247255cdfb649..57074cb13aa63 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionIdentifierIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionIdentifierIT.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java index 95204c783ec21..8de2d3c7e0024 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -255,8 +256,7 @@ public void testIllegalFilePath() { String errorMsg1 = TSStatusCode.SQL_PARSE_ERROR.getStatusCode() - + ": Error occurred while parsing SQL to physical plan: " - + "line 1:7 mismatched input 'path' expecting STRING_LITERAL"; + + ": Error occurred while parsing SQL to physical plan: line 1:7 no viable alternative at input 'REMOVE path'"; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("REMOVE path"); @@ -322,7 +322,7 @@ public void testUDFClassName() { // executed correctly try (ResultSet resultSet = statement.executeQuery("show functions")) { - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); int count = 0; while (resultSet.next()) { StringBuilder stringBuilder = new StringBuilder(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationByLevelIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationByLevelIT.java index 5f0beea7dccc3..19c76c7c4f2cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationByLevelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationByLevelIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it.aggregation; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java index 8eb1026121aac..524d0770a6c07 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java @@ -59,7 +59,6 @@ public class IoTDBAggregationIT { private static final double DETLA = 1e-6; - private static final String TIMESTAMP_STR = "Time"; private static final String TEMPERATURE_STR = "root.ln.wf01.wt01.temperature"; private static final String[] creationSqls = @@ -135,7 +134,7 @@ public static void tearDown() throws Exception { // details in: https://issues.apache.org/jira/projects/IOTDB/issues/IOTDB-54 @Test public void test() { - String[] retArray = new String[] {"0,2", "0,4", "0,3"}; + String[] retArray = new String[] {"2", "4", "3"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -145,10 +144,7 @@ public void test() { "SELECT count(temperature) FROM root.ln.wf01.wt01 WHERE time > 3")) { cnt = 0; while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(TEMPERATURE_STR)); + String ans = resultSet.getString(count(TEMPERATURE_STR)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -160,10 +156,7 @@ public void test() { "SELECT count(temperature) FROM root.ln.wf01.wt01 WHERE time > 3 order by time desc")) { cnt = 0; while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(TEMPERATURE_STR)); + String ans = resultSet.getString(count(TEMPERATURE_STR)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -174,10 +167,7 @@ public void test() { statement.executeQuery( "SELECT min_time(temperature) FROM root.ln.wf01.wt01 WHERE time > 3")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(TEMPERATURE_STR)); + String ans = resultSet.getString(minTime(TEMPERATURE_STR)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -188,10 +178,7 @@ public void test() { statement.executeQuery( "SELECT min_time(temperature) FROM root.ln.wf01.wt01 WHERE temperature > 3")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(TEMPERATURE_STR)); + String ans = resultSet.getString(minTime(TEMPERATURE_STR)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -213,7 +200,7 @@ public void test() { public void countTest() { String[] retArray = new String[] { - "0,2001,2001,2001,2001,2001,2001,2001,2001", "0,7500,7500,7500,7500,7500,7500,7500,7500" + "2001,2001,2001,2001,2001,2001,2001,2001", "7500,7500,7500,7500,7500,7500,7500,7500" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -226,9 +213,7 @@ public void countTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -256,9 +241,7 @@ public void countTest() { + "FROM root.vehicle.d0")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -287,9 +270,7 @@ public void countTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -316,9 +297,7 @@ public void countTest() { + "FROM root.vehicle.d0 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -348,8 +327,8 @@ public void countTest() { public void firstTest() { String[] retArray = new String[] { - "0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,2000", - "0,500,500,500.0,500,500,0x0500,1500-01-01,500" + "2000,2000,2000.0,2000,2000,0x2000,2000-01-01,1970-01-01T00:00:02.000Z", + "500,500,500.0,500,500,0x0500,1500-01-01,1970-01-01T00:00:00.500Z" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -361,9 +340,7 @@ public void firstTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -390,9 +367,7 @@ public void firstTest() { + "FROM root.vehicle.d0")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -421,9 +396,7 @@ public void firstTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -453,9 +426,9 @@ public void firstTest() { public void lastTest() { String[] retArray = new String[] { - "0,8499,8499.0,8499,0x8499,8499-01-01,8499", - "0,1499,1499.0,1499,0x1499,1499-01-01,1499", - "0,2200,2200.0,2200,0x2200,2200-01-01,2200" + "8499,8499.0,8499,0x8499,8499-01-01,1970-01-01T00:00:08.499Z", + "1499,1499.0,1499,0x1499,1499-01-01,1970-01-01T00:00:01.499Z", + "2200,2200.0,2200,0x2200,2200-01-01,1970-01-01T00:00:02.200Z" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -468,9 +441,7 @@ public void lastTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s2)) + "," @@ -493,9 +464,7 @@ public void lastTest() { + "FROM root.vehicle.d0 WHERE time <= 1600")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s2)) + "," @@ -518,9 +487,7 @@ public void lastTest() { + "FROM root.vehicle.d0 WHERE time <= 2200")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s2)) + "," @@ -544,9 +511,7 @@ public void lastTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s2)) + "," @@ -571,9 +536,7 @@ public void lastTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s2)) + "," @@ -597,8 +560,7 @@ public void lastTest() { @Test public void maxminTimeTest() { - String[] retArray = - new String[] {"0,8499,500,8499,500,8499,500,8499,500,8499,500", "0,2499,2000"}; + String[] retArray = new String[] {"8499,500,8499,500,8499,500,8499,500,8499,500", "2499,2000"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -610,9 +572,7 @@ public void maxminTimeTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime(d0s0)) + resultSet.getString(maxTime(d0s0)) + "," + resultSet.getString(minTime(d0s2)) + "," @@ -643,11 +603,7 @@ public void maxminTimeTest() { + "FROM root.vehicle.d0 WHERE time <= 2500 AND time > 1800")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime(d0s0)) - + "," - + resultSet.getString(minTime(d0s2)); + resultSet.getString(maxTime(d0s0)) + "," + resultSet.getString(minTime(d0s2)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -662,9 +618,7 @@ public void maxminTimeTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime(d0s0)) + resultSet.getString(maxTime(d0s0)) + "," + resultSet.getString(minTime(d0s2)) + "," @@ -698,7 +652,7 @@ public void maxminTimeTest() { public void firstLastValueTest() throws SQLException { String[] retArray = new String[] { - "0,2.2,4.4", + "2.2,4.4", }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -710,12 +664,7 @@ public void firstLastValueTest() throws SQLException { + "FROM root.ln.wf01.wt01 WHERE time > 1 AND time < 5")) { cnt = 0; while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(1) - + "," - + resultSet.getString(2); + String ans = resultSet.getString(1) + "," + resultSet.getString(2); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -727,7 +676,10 @@ public void firstLastValueTest() throws SQLException { @Test public void maxminValueTest() { String[] retArray = - new String[] {"0,8499,500.0,999,1000,8499-01-01,1000-01-01,8499,500", "0,2499,500.0"}; + new String[] { + "8499,500.0,999,1000,8499-01-01,1000-01-01,1970-01-01T00:00:08.499Z,1970-01-01T00:00:00.500Z", + "2499,500.0" + }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -739,9 +691,7 @@ public void maxminValueTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(minValue(d0s2)) + "," @@ -767,11 +717,7 @@ public void maxminValueTest() { "SELECT max_value(s0),min_value(s2) " + "FROM root.vehicle.d0 WHERE time < 2500")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) - + "," - + resultSet.getString(minValue(d0s2)); + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(minValue(d0s2)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -786,9 +732,7 @@ public void maxminValueTest() { + "FROM root.vehicle.d0 WHERE time >= 100 AND time < 9000 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(minValue(d0s2)) + "," @@ -817,8 +761,8 @@ public void maxminValueTest() { @Test public void avgSumTest() { double[][] retArray = { - {0.0, 1.4508E7, 7250.374812593702}, - {0.0, 626750.0, 1250.9980039920158} + {1.4508E7, 7250.374812593702}, + {626750.0, 1250.9980039920158} }; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -830,10 +774,9 @@ public void avgSumTest() { "SELECT sum(s0),avg(s2)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000")) { while (resultSet.next()) { - double[] ans = new double[3]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); + double[] ans = new double[2]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avg(d0s2))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -845,10 +788,9 @@ public void avgSumTest() { "SELECT sum(s0),avg(s2)" + "FROM root.vehicle.d0 WHERE time >= 1000 AND time <= 2000")) { while (resultSet.next()) { - double[] ans = new double[3]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); + double[] ans = new double[2]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avg(d0s2))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -862,10 +804,9 @@ public void avgSumTest() { "SELECT sum(s0),avg(s2)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000 order by time desc")) { while (resultSet.next()) { - double[] ans = new double[3]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); + double[] ans = new double[2]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avg(d0s2))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -943,10 +884,10 @@ public void avgSumErrorTest() { @Test public void mergeAggrOnOneSeriesTest() { double[][] retArray = { - {0.0, 1.4508E7, 7250.374812593702, 7250.374812593702, 1.4508E7}, - {0.0, 626750.0, 1250.9980039920158, 1250.9980039920158, 626750.0}, - {0.0, 1.4508E7, 2001, 7250.374812593702, 7250.374812593702}, - {0.0, 1.4508E7, 2001, 7250.374812593702, 7250.374812593702, 2001, 1.4508E7} + {1.4508E7, 7250.374812593702, 7250.374812593702, 1.4508E7}, + {626750.0, 1250.9980039920158, 1250.9980039920158, 626750.0}, + {1.4508E7, 2001, 7250.374812593702, 7250.374812593702}, + {1.4508E7, 2001, 7250.374812593702, 7250.374812593702, 2001, 1.4508E7} }; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -958,12 +899,11 @@ public void mergeAggrOnOneSeriesTest() { "SELECT sum(s0), avg(s2), avg(s0), sum(s2)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000")) { while (resultSet.next()) { - double[] ans = new double[5]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); - ans[3] = Double.parseDouble(resultSet.getString(avg(d0s0))); - ans[4] = Double.parseDouble(resultSet.getString(sum(d0s2))); + double[] ans = new double[4]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avg(d0s2))); + ans[2] = Double.parseDouble(resultSet.getString(avg(d0s0))); + ans[3] = Double.parseDouble(resultSet.getString(sum(d0s2))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -975,12 +915,11 @@ public void mergeAggrOnOneSeriesTest() { "SELECT sum(s0), avg(s2), avg(s0), sum(s2)" + "FROM root.vehicle.d0 WHERE time >= 1000 AND time <= 2000")) { while (resultSet.next()) { - double[] ans = new double[5]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); - ans[3] = Double.parseDouble(resultSet.getString(avg(d0s0))); - ans[4] = Double.parseDouble(resultSet.getString(sum(d0s2))); + double[] ans = new double[4]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avg(d0s2))); + ans[2] = Double.parseDouble(resultSet.getString(avg(d0s0))); + ans[3] = Double.parseDouble(resultSet.getString(sum(d0s2))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -992,12 +931,11 @@ public void mergeAggrOnOneSeriesTest() { "SELECT sum(s0), count(s0), avg(s2), avg(s0)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000")) { while (resultSet.next()) { - double[] ans = new double[5]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(count(d0s0))); - ans[3] = Double.parseDouble(resultSet.getString(avg(d0s2))); - ans[4] = Double.parseDouble(resultSet.getString(avg(d0s0))); + double[] ans = new double[4]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(count(d0s0))); + ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); + ans[3] = Double.parseDouble(resultSet.getString(avg(d0s0))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -1009,14 +947,13 @@ public void mergeAggrOnOneSeriesTest() { "SELECT sum(s2), count(s0), avg(s2), avg(s1), count(s2),sum(s0)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000")) { while (resultSet.next()) { - double[] ans = new double[7]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sum(d0s2))); - ans[2] = Double.parseDouble(resultSet.getString(count(d0s0))); - ans[3] = Double.parseDouble(resultSet.getString(avg(d0s2))); - ans[4] = Double.parseDouble(resultSet.getString(avg(d0s1))); - ans[5] = Double.parseDouble(resultSet.getString(count(d0s2))); - ans[6] = Double.parseDouble(resultSet.getString(sum(d0s0))); + double[] ans = new double[6]; + ans[0] = Double.parseDouble(resultSet.getString(sum(d0s2))); + ans[1] = Double.parseDouble(resultSet.getString(count(d0s0))); + ans[2] = Double.parseDouble(resultSet.getString(avg(d0s2))); + ans[3] = Double.parseDouble(resultSet.getString(avg(d0s1))); + ans[4] = Double.parseDouble(resultSet.getString(count(d0s2))); + ans[5] = Double.parseDouble(resultSet.getString(sum(d0s0))); assertArrayEquals(retArray[cnt], ans, DETLA); cnt++; } @@ -1032,7 +969,7 @@ public void mergeAggrOnOneSeriesTest() { public void descAggregationWithUnseqData() { String[] retArray = new String[] { - "0,12", + "12", }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -1041,10 +978,7 @@ public void descAggregationWithUnseqData() { try (ResultSet resultSet = statement.executeQuery("SELECT max_time(s1) FROM root.sg.d1 where time < 15")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime("root.sg.d1.s1")); + String ans = resultSet.getString(maxTime("root.sg.d1.s1")); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -1230,7 +1164,7 @@ public void noDataRegionTest() { @Test public void maxByTest() { - String[] retArray = new String[] {"0,8499", "0,2499", "0,8499"}; + String[] retArray = new String[] {"8499", "2499", "8499"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -1241,8 +1175,7 @@ public void maxByTest() { + "FROM root.vehicle.d0 WHERE time >= 100 AND time < 9000")) { cnt = 0; while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(maxBy("Time", d0s0)); + String ans = resultSet.getString(maxBy("Time", d0s0)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -1252,8 +1185,7 @@ public void maxByTest() { try (ResultSet resultSet = statement.executeQuery("SELECT max_by(time,s0) FROM root.vehicle.d0 WHERE time < 2500")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(maxBy("Time", d0s0)); + String ans = resultSet.getString(maxBy("Time", d0s0)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -1266,8 +1198,7 @@ public void maxByTest() { statement.executeQuery( "SELECT max_by(time,s0) FROM root.vehicle.d0 WHERE time >= 100 AND time < 9000 order by time desc")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(maxBy("Time", d0s0)); + String ans = resultSet.getString(maxBy("Time", d0s0)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -1281,7 +1212,7 @@ public void maxByTest() { @Test public void minByTest() { - String[] retArray = new String[] {"0,500", "0,500", "0,500"}; + String[] retArray = new String[] {"500", "500", "500"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -1292,8 +1223,7 @@ public void minByTest() { + "FROM root.vehicle.d0 WHERE time >= 100 AND time < 9000")) { cnt = 0; while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(minBy("Time", d0s0)); + String ans = resultSet.getString(minBy("Time", d0s0)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -1303,8 +1233,7 @@ public void minByTest() { try (ResultSet resultSet = statement.executeQuery("SELECT min_by(time,s0) FROM root.vehicle.d0 WHERE time < 2500")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(minBy("Time", d0s0)); + String ans = resultSet.getString(minBy("Time", d0s0)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -1317,8 +1246,7 @@ public void minByTest() { statement.executeQuery( "SELECT min_by(time,s0) FROM root.vehicle.d0 WHERE time >= 100 AND time < 9000 order by time desc")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(minBy("Time", d0s0)); + String ans = resultSet.getString(minBy("Time", d0s0)); Assert.assertEquals(retArray[cnt], ans); cnt++; } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java index 4d24e7284affb..743feb996db69 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java @@ -51,7 +51,6 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBAggregationLargeDataIT { - private static final String TIMESTAMP_STR = "Time"; private final String d0s0 = "root.vehicle.d0.s0"; private final String d0s1 = "root.vehicle.d0.s1"; private final String d0s2 = "root.vehicle.d0.s2"; @@ -127,7 +126,7 @@ public static void tearDown() throws Exception { @Test public void lastValueAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,9,39,63.0,E,true"}; + String[] retArray = new String[] {"9,39,63.0,E,true"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -139,9 +138,7 @@ public void lastValueAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s1)) + "," @@ -163,9 +160,7 @@ public void lastValueAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s1)) + "," @@ -187,7 +182,7 @@ public void lastValueAggreWithSingleFilterTest() { @Test public void sumAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,55061.0,156752.0,20254"}; + String[] retArray = new String[] {"55061.0,156752.0,20254"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -198,9 +193,7 @@ public void sumAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) + resultSet.getString(sum(d0s0)) + "," + resultSet.getString(sum(d0s1)) + "," @@ -218,9 +211,7 @@ public void sumAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) + resultSet.getString(sum(d0s0)) + "," + resultSet.getString(sum(d0s1)) + "," @@ -238,7 +229,7 @@ public void sumAggreWithSingleFilterTest() { @Test public void firstAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,90,1101,2.22,ddddd,true"}; + String[] retArray = new String[] {"90,1101,2.22,ddddd,true"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -250,9 +241,7 @@ public void firstAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -275,9 +264,7 @@ public void firstAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -299,7 +286,7 @@ public void firstAggreWithSingleFilterTest() { @Test public void avgAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,75,212,28"}; + String[] retArray = new String[] {"75,212,28"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -309,9 +296,7 @@ public void avgAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + Math.round(resultSet.getDouble(avg(d0s0))) + Math.round(resultSet.getDouble(avg(d0s0))) + "," + Math.round(resultSet.getDouble(avg(d0s1))) + "," @@ -328,9 +313,7 @@ public void avgAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + Math.round(resultSet.getDouble(avg(d0s0))) + Math.round(resultSet.getDouble(avg(d0s0))) + "," + Math.round(resultSet.getDouble(avg(d0s1))) + "," @@ -348,7 +331,7 @@ public void avgAggreWithSingleFilterTest() { @Test public void countAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,733,740,734"}; + String[] retArray = new String[] {"733,740,734"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -359,9 +342,7 @@ public void countAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -378,9 +359,7 @@ public void countAggreWithSingleFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -398,7 +377,7 @@ public void countAggreWithSingleFilterTest() { @Test public void minTimeAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,104,1,2,101,100"}; + String[] retArray = new String[] {"104,1,2,101,100"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -410,9 +389,7 @@ public void minTimeAggreWithSingleFilterTest() { + " from root.vehicle.d0 where s1 >= 0")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(d0s0)) + resultSet.getString(minTime(d0s0)) + "," + resultSet.getString(minTime(d0s1)) + "," @@ -435,9 +412,7 @@ public void minTimeAggreWithSingleFilterTest() { + " from root.vehicle.d0 where s1 >= 0 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(d0s0)) + resultSet.getString(minTime(d0s0)) + "," + resultSet.getString(minTime(d0s1)) + "," @@ -459,7 +434,7 @@ public void minTimeAggreWithSingleFilterTest() { @Test public void minValueAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,0,0,0.0"}; + String[] retArray = new String[] {"0,0,0.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -471,9 +446,7 @@ public void minValueAggreWithSingleFilterTest() { + "where s1 < 50000 and s1 != 100")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minValue(d0s0)) + resultSet.getString(minValue(d0s0)) + "," + resultSet.getString(minValue(d0s1)) + "," @@ -491,9 +464,7 @@ public void minValueAggreWithSingleFilterTest() { + "where s1 < 50000 and s1 != 100 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minValue(d0s0)) + resultSet.getString(minValue(d0s0)) + "," + resultSet.getString(minValue(d0s1)) + "," @@ -511,7 +482,7 @@ public void minValueAggreWithSingleFilterTest() { @Test public void maxValueAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,99,40000,122.0"}; + String[] retArray = new String[] {"99,40000,122.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -523,9 +494,7 @@ public void maxValueAggreWithSingleFilterTest() { + "where s1 < 50000 and s1 != 100")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(maxValue(d0s1)) + "," @@ -543,9 +512,7 @@ public void maxValueAggreWithSingleFilterTest() { + "where s1 < 50000 and s1 != 100 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(maxValue(d0s1)) + "," @@ -563,7 +530,7 @@ public void maxValueAggreWithSingleFilterTest() { @Test public void extremeAggreWithSingleFilterTest() { - String[] retArray = new String[] {"0,99,40000,122.0"}; + String[] retArray = new String[] {"99,40000,122.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -576,9 +543,7 @@ public void extremeAggreWithSingleFilterTest() { + "where s1 < 50000 and s1 != 100")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(extreme(d0s0)) + resultSet.getString(extreme(d0s0)) + "," + resultSet.getString(extreme(d0s1)) + "," @@ -597,9 +562,7 @@ public void extremeAggreWithSingleFilterTest() { + "where s1 < 50000 and s1 != 100 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(extreme(d0s0)) + resultSet.getString(extreme(d0s0)) + "," + resultSet.getString(extreme(d0s1)) + "," @@ -617,7 +580,7 @@ public void extremeAggreWithSingleFilterTest() { @Test public void avgAggreWithMultiFilterTest() { - String[] retArray = new String[] {"0,55061.0,733,75,212,28"}; + String[] retArray = new String[] {"55061.0,733,75,212,28"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -629,9 +592,7 @@ public void avgAggreWithMultiFilterTest() { + "avg(s2) from root.vehicle.d0 where s1 >= 0 or s2 < 10")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) + resultSet.getString(sum(d0s0)) + "," + resultSet.getString(count(d0s0)) + "," @@ -653,7 +614,7 @@ public void avgAggreWithMultiFilterTest() { @Test public void sumAggreWithMultiFilterTest() { - String[] retArray = new String[] {"0,55061.0,156752.0,20262"}; + String[] retArray = new String[] {"55061.0,156752.0,20262"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -664,9 +625,7 @@ public void sumAggreWithMultiFilterTest() { "select sum(s0),sum(s1),sum(s2) from root.vehicle.d0 where s1 >= 0 or s2 < 10")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) + resultSet.getString(sum(d0s0)) + "," + resultSet.getString(sum(d0s1)) + "," @@ -684,9 +643,7 @@ public void sumAggreWithMultiFilterTest() { + " where s1 >= 0 or s2 < 10 order by time desc ")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) + resultSet.getString(sum(d0s0)) + "," + resultSet.getString(sum(d0s1)) + "," @@ -704,7 +661,7 @@ public void sumAggreWithMultiFilterTest() { @Test public void firstAggreWithMultiFilterTest() { - String[] retArray = new String[] {"0,90,1101,2.22,ddddd,true"}; + String[] retArray = new String[] {"90,1101,2.22,ddddd,true"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -716,9 +673,7 @@ public void firstAggreWithMultiFilterTest() { + "first_value(s4) from root.vehicle.d0 where s1 >= 0 or s2 < 10")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -741,9 +696,7 @@ public void firstAggreWithMultiFilterTest() { + "first_value(s4) from root.vehicle.d0 where s1 >= 0 or s2 < 10 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)) + "," @@ -765,7 +718,7 @@ public void firstAggreWithMultiFilterTest() { @Test public void countAggreWithMultiFilterTest() { - String[] retArray = new String[] {"0,733,740,736,482,1"}; + String[] retArray = new String[] {"733,740,736,482,1"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -777,9 +730,7 @@ public void countAggreWithMultiFilterTest() { + "count(s4) from root.vehicle.d0 where s1 >= 0 or s2 < 10")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -801,7 +752,7 @@ public void countAggreWithMultiFilterTest() { @Test public void maxTimeAggreWithMultiFilterTest() { - String[] retArray = new String[] {"0,3999,3999,3999,3599,100"}; + String[] retArray = new String[] {"3999,3999,3999,3599,100"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -814,9 +765,7 @@ public void maxTimeAggreWithMultiFilterTest() { + "where s1 < 50000 and s1 != 100")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime(d0s0)) + resultSet.getString(maxTime(d0s0)) + "," + resultSet.getString(maxTime(d0s1)) + "," @@ -839,9 +788,7 @@ public void maxTimeAggreWithMultiFilterTest() { + "where s1 < 50000 and s1 != 100 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime(d0s0)) + resultSet.getString(maxTime(d0s0)) + "," + resultSet.getString(maxTime(d0s1)) + "," diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationSmallDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationSmallDataIT.java index aec2ae3a48f21..7ed236411dfc1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationSmallDataIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationSmallDataIT.java @@ -129,7 +129,7 @@ public static void tearDown() throws Exception { @Test public void countWithTimeFilterTest() { - String[] retArray = new String[] {"0,3,7,4,5,1"}; + String[] retArray = new String[] {"3,7,4,5,1"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -141,9 +141,7 @@ public void countWithTimeFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -165,7 +163,7 @@ public void countWithTimeFilterTest() { @Test public void countWithoutFilterTest() { - String[] retArray = new String[] {"0,4,0,6,1"}; + String[] retArray = new String[] {"4,0,6,1"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -177,9 +175,7 @@ public void countWithoutFilterTest() { "SELECT count(d0.s0),count(d1.s1),count(d0.s3),count(d0.s4) FROM root.vehicle")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d1s1)) + "," @@ -199,7 +195,7 @@ public void countWithoutFilterTest() { @Test public void maxValueWithoutFilterTest() { - String[] retArray = new String[] {"0,22222,null"}; + String[] retArray = new String[] {"22222,null"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -220,11 +216,7 @@ public void maxValueWithoutFilterTest() { statement.executeQuery("SELECT max_value(d0.s0),max_value(d1.s1) FROM root.vehicle")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) - + "," - + resultSet.getString(maxValue(d1s1)); + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(maxValue(d1s1)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -238,7 +230,7 @@ public void maxValueWithoutFilterTest() { @Test public void extremeWithoutFilterTest() { - String[] retArray = new String[] {"0,22222,null"}; + String[] retArray = new String[] {"22222,null"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -259,11 +251,7 @@ public void extremeWithoutFilterTest() { statement.executeQuery("SELECT extreme(d0.s0),extreme(d1.s1) FROM root.vehicle")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(extreme(d0s0)) - + "," - + resultSet.getString(extreme(d1s1)); + resultSet.getString(extreme(d0s0)) + "," + resultSet.getString(extreme(d1s1)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -277,7 +265,7 @@ public void extremeWithoutFilterTest() { @Test public void firstValueWithoutFilterTest() { - String[] retArray = new String[] {"0,90,null,aaaaa"}; + String[] retArray = new String[] {"90,null,aaaaa"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -289,9 +277,7 @@ public void firstValueWithoutFilterTest() { "SELECT first_value(d0.s0),first_value(d1.s1),first_value(d0.s3) FROM root.vehicle")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d1s1)) + "," @@ -309,7 +295,7 @@ public void firstValueWithoutFilterTest() { @Test public void lastValueWithoutFilterTest() { - String[] retArray = new String[] {"0,22222,null,good"}; + String[] retArray = new String[] {"22222,null,good"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -321,9 +307,7 @@ public void lastValueWithoutFilterTest() { "SELECT last_value(d0.s0),last_value(d1.s1),last_value(d0.s3) FROM root.vehicle")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d1s1)) + "," @@ -341,7 +325,7 @@ public void lastValueWithoutFilterTest() { @Test public void sumWithoutFilterTest() { - String[] retArray = new String[] {"0,22610.0,null"}; + String[] retArray = new String[] {"22610.0,null"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -351,12 +335,7 @@ public void sumWithoutFilterTest() { try (ResultSet resultSet = statement.executeQuery("SELECT sum(d0.s0),sum(d1.s1) FROM root.vehicle")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) - + "," - + resultSet.getString(sum(d1s1)); + String ans = resultSet.getString(sum(d0s0)) + "," + resultSet.getString(sum(d1s1)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -370,7 +349,7 @@ public void sumWithoutFilterTest() { @Test public void lastValueWithSingleValueFilterTest() { - String[] retArray = new String[] {"0,22222,55555"}; + String[] retArray = new String[] {"22222,55555"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -380,11 +359,7 @@ public void lastValueWithSingleValueFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(lastValue(d0s0)) - + "," - + resultSet.getString(lastValue(d0s1)); + resultSet.getString(lastValue(d0s0)) + "," + resultSet.getString(lastValue(d0s1)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -398,7 +373,7 @@ public void lastValueWithSingleValueFilterTest() { @Test public void firstValueWithSingleValueFilterTest() { - String[] retArray = new String[] {"0,99,180"}; + String[] retArray = new String[] {"99,180"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -408,11 +383,7 @@ public void firstValueWithSingleValueFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(firstValue(d0s0)) - + "," - + resultSet.getString(firstValue(d0s1)); + resultSet.getString(firstValue(d0s0)) + "," + resultSet.getString(firstValue(d0s1)); Assert.assertEquals(retArray[cnt], ans); cnt++; } @@ -426,7 +397,7 @@ public void firstValueWithSingleValueFilterTest() { @Test public void sumWithSingleValueFilterTest() { - String[] retArray = new String[] {"0,22321.0,55934.0,1029"}; + String[] retArray = new String[] {"22321.0,55934.0,1029"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -436,9 +407,7 @@ public void sumWithSingleValueFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(sum(d0s0)) + resultSet.getString(sum(d0s0)) + "," + resultSet.getString(sum(d0s1)) + "," @@ -456,7 +425,7 @@ public void sumWithSingleValueFilterTest() { @Test public void avgWithSingleValueFilterTest() { - String[] retArray = new String[] {"0,11160.5,18645,206"}; + String[] retArray = new String[] {"11160.5,18645,206"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -466,9 +435,7 @@ public void avgWithSingleValueFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(avg(d0s0)) + resultSet.getString(avg(d0s0)) + "," + Math.round(resultSet.getDouble(avg(d0s1))) + "," @@ -486,7 +453,7 @@ public void avgWithSingleValueFilterTest() { @Test public void countWithSingleValueFilterTest() { - String[] retArray = new String[] {"0,2,3,5,1,0"}; + String[] retArray = new String[] {"2,3,5,1,0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -498,9 +465,7 @@ public void countWithSingleValueFilterTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count(d0s0)) + resultSet.getString(count(d0s0)) + "," + resultSet.getString(count(d0s1)) + "," @@ -522,7 +487,7 @@ public void countWithSingleValueFilterTest() { @Test public void minTimeWithMultiValueFiltersTest() { - String[] retArray = new String[] {"0,104,1,2,101,100"}; + String[] retArray = new String[] {"104,1,2,101,100"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -534,9 +499,7 @@ public void minTimeWithMultiValueFiltersTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minTime(d0s0)) + resultSet.getString(minTime(d0s0)) + "," + resultSet.getString(minTime(d0s1)) + "," @@ -558,7 +521,7 @@ public void minTimeWithMultiValueFiltersTest() { @Test public void maxTimeWithMultiValueFiltersTest() { - String[] retArray = new String[] {"0,105,105,105,102,100"}; + String[] retArray = new String[] {"105,105,105,102,100"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -570,9 +533,7 @@ public void maxTimeWithMultiValueFiltersTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxTime(d0s0)) + resultSet.getString(maxTime(d0s0)) + "," + resultSet.getString(maxTime(d0s1)) + "," @@ -594,7 +555,7 @@ public void maxTimeWithMultiValueFiltersTest() { @Test public void minValueWithMultiValueFiltersTest() { - String[] retArray = new String[] {"0,90,180,2.22"}; + String[] retArray = new String[] {"90,180,2.22"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -606,9 +567,7 @@ public void minValueWithMultiValueFiltersTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(minValue(d0s0)) + resultSet.getString(minValue(d0s0)) + "," + resultSet.getString(minValue(d0s1)) + "," @@ -627,7 +586,7 @@ public void minValueWithMultiValueFiltersTest() { @Test public void maxValueWithMultiValueFiltersTest() { - String[] retArray = new String[] {"0,99,40000,11.11"}; + String[] retArray = new String[] {"99,40000,11.11"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -639,9 +598,7 @@ public void maxValueWithMultiValueFiltersTest() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(maxValue(d0s0)) + resultSet.getString(maxValue(d0s0)) + "," + resultSet.getString(maxValue(d0s1)) + "," @@ -659,7 +616,7 @@ public void maxValueWithMultiValueFiltersTest() { @Test public void extremeWithMultiValueFiltersTest() throws ClassNotFoundException { - String[] retArray = new String[] {"0,99,40000,11.11"}; + String[] retArray = new String[] {"99,40000,11.11"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -672,9 +629,7 @@ public void extremeWithMultiValueFiltersTest() throws ClassNotFoundException { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(extreme(d0s0)) + resultSet.getString(extreme(d0s0)) + "," + resultSet.getString(extreme(d0s1)) + "," diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java index bc82992065bbd..3f45ccdd58fd6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java @@ -208,6 +208,19 @@ public void testContIfWithoutTransform() { resultSetEqualTest("select Count_if(s3, 1) from root.db.d1", expectedHeader, retArray); } + @Test + public void testMultiAttributes() { + String[] expectedHeader = + new String[] { + "Count_if(root.db.d1.s3, 1, \"attr1\"=\"1\", \"attr2\"=\"2\", \"attr3\"=\"3\")" + }; + String[] retArray = new String[] {"1,"}; + resultSetEqualTest( + "select Count_if(s3, 1, \"attr1\"=\"1\",\"attr2\"=\"2\",\"attr3\"=\"3\") from root.db.d1", + expectedHeader, + retArray); + } + @Test public void testContIfWithGroupByLevel() { String[] expectedHeader = new String[] {"Count_if(root.db.*.s1 = 0 & root.db.*.s2 = 0, 3)"}; @@ -253,7 +266,7 @@ public void testContIfWithGroupByLevel() { public void testContIfWithSlidingWindow() { assertTestFail( "select count_if(s1>1,1) from root.db.d1 group by time([1,10),3ms,2ms)", - TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": COUNT_IF with slidingWindow is not supported now"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java index 49bee9845970a..5cceaa67887ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java @@ -161,7 +161,7 @@ public void testModeWithGroupByLevel() { public void testModeWithSlidingWindow() { assertTestFail( "select mode(s1) from root.db.d1 group by time([1,10),3ms,2ms)", - TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": MODE with slidingWindow is not supported now"); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java index 0be9aa7410794..056797e204aab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java @@ -362,7 +362,7 @@ public void testTimeDurationManyColumnOnManyDataRegion() { public void testTimeDurationWithSlidingWindow() { assertTestFail( "select time_duration(s2) from root.db1.d2 group by ([1,10),5ms,2ms)", - TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": TIME_DURATION with slidingWindow is not supported now"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java index ccf435f836205..1ccdc8c1195e2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java @@ -210,7 +210,8 @@ public void testMaxByWithDifferentXAndYInputTypes() { "root.db.d1", new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"}, new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"}); - String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"}; + String[] retArray = + new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"}; for (Map.Entry expectedHeader : expectedHeaders.entrySet()) { String y = expectedHeader.getKey(); resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java index 77cb86e62d4d1..82867e782ccd4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java @@ -210,7 +210,8 @@ public void testMaxByWithDifferentXAndYInputTypes() { "root.db.d1", new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"}, new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"}); - String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"}; + String[] retArray = + new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"}; for (Map.Entry expectedHeader : expectedHeaders.entrySet()) { String y = expectedHeader.getKey(); resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java index cc5fdeb5b27fe..1bb3e76061dab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java @@ -100,6 +100,7 @@ public class IoTDBAlignByDeviceIT { "insert into root.vehicle.d1(timestamp,s0) values(1,999)", "insert into root.vehicle.d1(timestamp,s0) values(1000,888)", "insert into root.other.d1(timestamp,s0) values(2, 3.14)", + "insert into root.other.d2(timestamp,s6) values(6, 6.66)", }; @BeforeClass @@ -1184,4 +1185,48 @@ public void removeDeviceWhereMeasurementWhenNoDeviceSelectTest() { + e.getMessage()); } } + + @Test + public void nonExistMeasurementInHavingTest() { + String[] retArray = + new String[] { + "1,root.other.d1,3.14,null,", "5,root.other.d2,null,6.66,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s0),last_value(s6) from root.other.** group by ([1,10),2ms) having last_value(s0) is not null or last_value(s6) is not null align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,Device,last_value(s0),last_value(s6)", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.FLOAT, Types.DOUBLE, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java index 05e2104b904d2..a71cb77371359 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java @@ -59,6 +59,9 @@ public class IoTDBAlignByDeviceWithTemplateIT { "INSERT INTO root.sg2.d4(timestamp,s1,s2,s3) values(1,1111.1,true,1111), (5,5555.5,false,5555);", }; + String[] expectedHeader; + String[] retArray; + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); @@ -70,11 +73,32 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void singleDeviceTest() { + expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + retArray = + new String[] { + "1,root.sg1.d1,1,1.1,false,", + }; + resultSetEqualTest( + "SELECT * FROM root.sg1.d1 order by time desc offset 1 limit 1 ALIGN BY DEVICE;", + expectedHeader, + retArray); + retArray = + new String[] { + "1,root.sg2.d1,1,1.1,false,", + }; + resultSetEqualTest( + "SELECT * FROM root.sg2.d1 order by time desc offset 1 limit 1 ALIGN BY DEVICE;", + expectedHeader, + retArray); + } + @Test public void selectWildcardNoFilterTest() { // 1. order by device - String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; - String[] retArray = + expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + retArray = new String[] { "1,root.sg1.d1,1,1.1,false,", "2,root.sg1.d1,2,2.2,false,", diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java index a595c5a96cbee..60cf1a5d5f0c1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java @@ -53,11 +53,30 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + String[] expectedHeader; + String[] retArray; + + @Test + public void singleDeviceTest() { + expectedHeader = new String[] {"Time,Device,precipitation"}; + retArray = new String[] {"1668960000200,root.weather.London,1667492178318,"}; + resultSetEqualTest( + "select precipitation from root.weather.London where precipitation>1667492178118 order by time offset 1 limit 1 align by device", + expectedHeader, + retArray); + + retArray = new String[] {"1668960000200,root.weather.London,1667492178318,"}; + resultSetEqualTest( + "select precipitation from root.weather.London where precipitation>1667492178118 order by precipitation offset 1 limit 1 align by device", + expectedHeader, + retArray); + } + @Test public void orderByCanNotPushLimitTest() { // 1. value filter, can not push down LIMIT - String[] expectedHeader = new String[] {"Time,Device,s1"}; - String[] retArray = new String[] {"3,root.db.d1,111,"}; + expectedHeader = new String[] {"Time,Device,s1"}; + retArray = new String[] {"3,root.db.d1,111,"}; resultSetEqualTest( "SELECT * FROM root.db.** WHERE s1>40 ORDER BY TIME LIMIT 1 ALIGN BY DEVICE;", expectedHeader, diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java index 43e37c597c97d..8d69a09e89725 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAggregationWithDeletion2IT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java index d8a04e08f1929..514757ccbaca5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java @@ -18,8 +18,8 @@ */ package org.apache.iotdb.db.it.aligned; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -41,7 +41,7 @@ import java.util.Set; import static org.apache.iotdb.itbase.constant.TestConstant.DATA_TYPE_STR; -import static org.apache.iotdb.itbase.constant.TestConstant.TIMESEIRES_STR; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESERIES_STR; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.VALUE_STR; import static org.junit.Assert.assertEquals; @@ -136,7 +136,7 @@ public void selectAllAlignedAndNonAlignedLastTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -171,7 +171,7 @@ public void selectAllAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -208,7 +208,7 @@ public void selectSomeAlignedLastTest1() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -241,7 +241,7 @@ public void selectSomeAlignedLastTest2() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -275,7 +275,7 @@ public void selectSomeAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -313,7 +313,7 @@ public void selectSomeAlignedAndNonAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -340,4 +340,118 @@ public void cacheHitTest() { selectSomeAlignedLastWithTimeFilterTest(); selectSomeAlignedAndNonAlignedLastWithTimeFilterTest(); } + + @Test + public void testNullInMemtable() { + String[] sqls = + new String[] { + "create aligned timeseries root.ln_1.tb_6141(fengjituichu_BOOLEAN BOOLEAN encoding=RLE,`chushuiNH4-N_DOUBLE` DOUBLE encoding=GORILLA,mochanshuizhuangtai_BOOLEAN BOOLEAN encoding=RLE,11_TEXT TEXT encoding=PLAIN,chanshuijianxieyunxingshijianshezhi_DOUBLE DOUBLE encoding=GORILLA,wenben_TEXT TEXT encoding=PLAIN, fengjitouru_BOOLEAN BOOLEAN encoding=RLE,meiju_INT32 INT32 encoding=RLE,chushuiTP_DOUBLE DOUBLE encoding=GORILLA,shuiguanliusu_DOUBLE DOUBLE encoding=GORILLA,CO2_DOUBLE DOUBLE encoding=GORILLA,`kaiguanliang-yunxing_BOOLEAN` BOOLEAN encoding=RLE,gongnengma_DOUBLE DOUBLE encoding=GORILLA);", + "alter timeseries root.ln_1.tb_6141.fengjituichu_BOOLEAN upsert alias=fengjituichu;", + "alter timeseries root.ln_1.tb_6141.shuiguanliusu_DOUBLE upsert alias=shuiguanliusu;", + "alter timeseries root.ln_1.tb_6141.CO2_DOUBLE upsert alias=CO2;", + "alter timeseries root.ln_1.tb_6141.fengjitouru_BOOLEAN upsert alias=fengjitouru;", + "alter timeseries root.ln_1.tb_6141.chanshuijianxieyunxingshijianshezhi_DOUBLE upsert alias=chanshuijianxieyunxingshijianshezhi;", + "alter timeseries root.ln_1.tb_6141.mochanshuizhuangtai_BOOLEAN upsert alias=mochanshuizhuangtai;", + "alter timeseries root.ln_1.tb_6141.meiju_INT32 upsert alias=meiju;", + "alter timeseries root.ln_1.tb_6141.chushuiTP_DOUBLE upsert alias=chushuiTP;", + "alter timeseries root.ln_1.tb_6141.wenben_TEXT upsert alias=wenben;", + "alter timeseries root.ln_1.tb_6141.`chushuiNH4-N_DOUBLE` upsert alias=`chushuiNH4-N`;", + "alter timeseries root.ln_1.tb_6141.gongnengma_DOUBLE upsert alias=gongnengma;", + "alter timeseries root.ln_1.tb_6141.11_TEXT upsert alias=`11`;", + "alter timeseries root.ln_1.tb_6141.`kaiguanliang-yunxing_BOOLEAN` upsert alias=`kaiguanliang-yunxing`;", + "insert into root.ln_1.tb_6141(time,chanshuijianxieyunxingshijianshezhi_DOUBLE) aligned values(1679365910000,10.0);", + "insert into root.ln_1.tb_6141(time,chushuiTP_DOUBLE) aligned values(1679365910000,15.0);", + "insert into root.ln_1.tb_6141(time,gongnengma_DOUBLE) aligned values(1679477545000,2.0);", + "insert into root.ln_1.tb_6141(time,wenben_TEXT) aligned values(1675995566000,52);", + "insert into root.ln_1.tb_6141(time,meiju_INT32) aligned values(1675995566000,2);", + "insert into root.ln_1.tb_6141(time,shuiguanliusu_DOUBLE) aligned values(1679365910000,15.0);", + "insert into root.ln_1.tb_6141(time,mochanshuizhuangtai_BOOLEAN) aligned values(1677033625000,true);", + "insert into root.ln_1.tb_6141(time,fengjitouru_BOOLEAN) aligned values(1675995566000,true);", + "insert into root.ln_1.tb_6141(time,fengjituichu_BOOLEAN) aligned values(1675995566000,false);", + "insert into root.ln_1.tb_6141(time,11_TEXT) aligned values(1679365910000,13);", + "insert into root.ln_1.tb_6141(time,CO2_DOUBLE) aligned values(1679365910000,12.0);", + "insert into root.ln_1.tb_6141(time,`chushuiNH4-N_DOUBLE`) aligned values(1679365910000,12.0);", + "insert into root.ln_1.tb_6141(time,`kaiguanliang-yunxing_BOOLEAN`) aligned values(1675995566000,false);", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // create aligned and non-aligned time series + for (String sql : sqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + Set retSet = + new HashSet<>( + Arrays.asList( + "1679477545000,root.ln_1.tb_6141.gongnengma_DOUBLE,2.0,DOUBLE", + "1675995566000,root.ln_1.tb_6141.wenben_TEXT,52,TEXT")); + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select last gongnengma,wenben from root.ln_1.tb_6141 order by timeseries asc;")) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(TIMESERIES_STR) + + "," + + resultSet.getString(VALUE_STR) + + "," + + resultSet.getString(DATA_TYPE_STR); + assertTrue(ans, retSet.contains(ans)); + cnt++; + } + assertEquals(retSet.size(), cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + retSet = + new HashSet<>( + Arrays.asList( + "1679477545000,root.ln_1.tb_6141.gongnengma_DOUBLE,2.0,DOUBLE", + "1677033625000,root.ln_1.tb_6141.mochanshuizhuangtai_BOOLEAN,true,BOOLEAN", + "1675995566000,root.ln_1.tb_6141.wenben_TEXT,52,TEXT")); + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select last gongnengma,mochanshuizhuangtai,wenben from root.ln_1.tb_6141 order by timeseries asc;")) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(TIMESERIES_STR) + + "," + + resultSet.getString(VALUE_STR) + + "," + + resultSet.getString(DATA_TYPE_STR); + assertTrue(ans, retSet.contains(ans)); + cnt++; + } + assertEquals(retSet.size(), cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQuery4IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQuery4IT.java index 6c3104d0aa309..b42e46ae032b4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQuery4IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQuery4IT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.aligned; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java index 016e2d59daab7..64ef126853127 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java @@ -18,8 +18,8 @@ */ package org.apache.iotdb.db.it.aligned; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.db.it.utils.AlignedWriteUtil; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java index dc26696be609e..f02e2fd6ec8ae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java @@ -40,7 +40,7 @@ import java.util.Set; import static org.apache.iotdb.itbase.constant.TestConstant.DATA_TYPE_STR; -import static org.apache.iotdb.itbase.constant.TestConstant.TIMESEIRES_STR; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESERIES_STR; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.VALUE_STR; import static org.junit.Assert.assertEquals; @@ -96,7 +96,7 @@ public void selectAllAlignedLastTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -137,7 +137,7 @@ public void selectAllAlignedAndNonAlignedLastTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -169,7 +169,7 @@ public void selectAllAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -203,7 +203,7 @@ public void selectSomeAlignedLastTest1() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -234,7 +234,7 @@ public void selectSomeAlignedLastTest2() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -267,7 +267,7 @@ public void selectSomeAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -303,7 +303,7 @@ public void selectSomeAlignedAndNonAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java index 202ecf448a44d..25b6a2098c9c3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java @@ -40,7 +40,7 @@ import java.util.Set; import static org.apache.iotdb.itbase.constant.TestConstant.DATA_TYPE_STR; -import static org.apache.iotdb.itbase.constant.TestConstant.TIMESEIRES_STR; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESERIES_STR; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.VALUE_STR; import static org.junit.Assert.assertEquals; @@ -90,7 +90,7 @@ public void selectAllAlignedLastTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -134,7 +134,7 @@ public void selectAllAlignedAndNonAlignedLastTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -169,7 +169,7 @@ public void selectAllAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -206,7 +206,7 @@ public void selectSomeAlignedLastTest1() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -239,7 +239,7 @@ public void selectSomeAlignedLastTest2() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -273,7 +273,7 @@ public void selectSomeAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," @@ -310,7 +310,7 @@ public void selectSomeAlignedAndNonAlignedLastWithTimeFilterTest() { String ans = resultSet.getString(TIMESTAMP_STR) + "," - + resultSet.getString(TIMESEIRES_STR) + + resultSet.getString(TIMESERIES_STR) + "," + resultSet.getString(VALUE_STR) + "," diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java index 1a65682fc5b83..f9639d3ea2d2c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java @@ -74,6 +74,15 @@ public void testAlignedRawDataAlignByTime1() { resultSetEqualTest( "select s2, s3 from root.sg1.d1 where s2 - 1 >= 9 and s2 < 30", expectedHeader1, retArray1); + resultSetEqualTest( + "select s2, s3 from root.sg1.d1 where 9 <= s2 - 1 and 30 > s2", expectedHeader1, retArray1); + + retArray1 = new String[] {"20,20,20,"}; + resultSetEqualTest( + "select s2, s3 from root.sg1.d1 where 9 <= s2 - 1 and 30 > s2 and 19 < time", + expectedHeader1, + retArray1); + String expectedHeader2 = "Time,root.sg1.d1.s3,"; String[] retArray2 = new String[] { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java index 93165a16ff6a6..f88c47bbe2e5e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java @@ -20,12 +20,17 @@ package org.apache.iotdb.db.it.auth; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.jdbc.IoTDBSQLException; +import org.apache.iotdb.rpc.TSStatusCode; +import com.google.common.collect.ImmutableList; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -45,7 +50,10 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.Callable; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -114,10 +122,11 @@ public void allPrivilegesTest() throws SQLException { // tempuser revoke his write_schema privilege userStmt.execute("REVOKE WRITE_SCHEMA ON root.** FROM USER tempuser"); - // 6. REVOKE ALL will get an error. Assert.assertThrows( - SQLException.class, - () -> adminStmt.execute("REVOKE ALL on root.** FROM USER tempuser")); + SQLException.class, () -> userStmt.execute("GRANT READ_DATA root.t1 to USER tempuser")); + + // 6. REVOKE ALL will be ok. + adminStmt.execute("REVOKE ALL on root.** FROM USER tempuser"); adminStmt.execute("GRANT ALL ON root.** TO USER tempuser"); adminStmt.execute("REVOKE ALL ON root.** FROM USER tempuser"); @@ -154,6 +163,30 @@ public void allPrivilegesTest() throws SQLException { Assert.assertThrows( SQLException.class, () -> userStmt.execute("GRANT USER tempuser PRIVILEGES WRITE_SCHEMA ON root.a")); + + adminStmt.execute("GRANT ALL ON root.** TO USER tempuser WITH GRANT OPTION"); + userStmt.execute("CREATE USER testuser 'password'"); + userStmt.execute("GRANT ALL ON root.** TO USER testuser WITH GRANT OPTION"); + ResultSet dataSet = userStmt.executeQuery("LIST PRIVILEGES OF USER testuser"); + + Set ansSet = + new HashSet<>( + Arrays.asList( + ",,MANAGE_USER,true,", + ",,MANAGE_ROLE,true,", + ",,USE_TRIGGER,true,", + ",,USE_UDF,true,", + ",,USE_CQ,true,", + ",,USE_PIPE,true,", + ",,USE_MODEL,true,", + ",,EXTEND_TEMPLATE,true,", + ",,MANAGE_DATABASE,true,", + ",,MAINTAIN,true,", + ",root.**,READ_DATA,true,", + ",root.**,WRITE_DATA,true,", + ",root.**,READ_SCHEMA,true,", + ",root.**,WRITE_SCHEMA,true,")); + TestUtils.assertResultSetEqual(dataSet, "Role,Scope,Privileges,GrantOption,", ansSet); } } } @@ -254,9 +287,7 @@ public void illegalGrantRevokeUserTest() throws SQLException { () -> userStmt.execute("GRANT WRITE_SCHEMA on root.a.b TO USER tempuser")); // revoke a non-existing privilege adminStmt.execute("REVOKE MANAGE_USER on root.** FROM USER tempuser"); - Assert.assertThrows( - SQLException.class, - () -> adminStmt.execute("REVOKE MANAGE_USER on root.** FROM USER tempuser")); + // revoke a non-existing user Assert.assertThrows( SQLException.class, @@ -312,9 +343,7 @@ public void createDeleteTimeSeriesTest() throws SQLException { () -> userStmt.execute("CREATE TIMESERIES root.b.a WITH DATATYPE=INT32,ENCODING=PLAIN")); - Assert.assertThrows( - SQLException.class, - () -> adminStmt.execute("REVOKE WRITE_SCHEMA ON root.a.b FROM USER tempuser")); + adminStmt.execute("REVOKE WRITE_SCHEMA ON root.a.b FROM USER tempuser"); // no privilege to create this one anymore Assert.assertThrows( SQLException.class, @@ -450,7 +479,7 @@ public void testListUser() throws SQLException { try { ResultSet resultSet = adminStmt.executeQuery("LIST USER"); - String ans = String.format("root,\n"); + String ans = "root,\n"; try { validateResultSet(resultSet, ans); @@ -551,7 +580,7 @@ public void testListUserRole() throws SQLException { // user1 : role1; MANAGE_ROLE,MANAGE_USER // user2 : role1, role2; ResultSet resultSet; - String ans = ""; + String ans; Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); Statement userStmt = userCon.createStatement(); try { @@ -618,16 +647,16 @@ public void testListUserPrivileges() throws SQLException { validateResultSet(resultSet, ans); resultSet = adminStmt.executeQuery("LIST PRIVILEGES OF USER root"); ans = - ",root.**,MANAGE_USER,true,\n" - + ",root.**,MANAGE_ROLE,true,\n" - + ",root.**,USE_TRIGGER,true,\n" - + ",root.**,USE_UDF,true,\n" - + ",root.**,USE_CQ,true,\n" - + ",root.**,USE_PIPE,true,\n" - + ",root.**,USE_MODEL,true,\n" - + ",root.**,EXTEND_TEMPLATE,true,\n" - + ",root.**,MANAGE_DATABASE,true,\n" - + ",root.**,MAINTAIN,true,\n" + ",,MANAGE_USER,true,\n" + + ",,MANAGE_ROLE,true,\n" + + ",,USE_TRIGGER,true,\n" + + ",,USE_UDF,true,\n" + + ",,USE_CQ,true,\n" + + ",,USE_PIPE,true,\n" + + ",,USE_MODEL,true,\n" + + ",,EXTEND_TEMPLATE,true,\n" + + ",,MANAGE_DATABASE,true,\n" + + ",,MAINTAIN,true,\n" + ",root.**,READ_DATA,true,\n" + ",root.**,WRITE_DATA,true,\n" + ",root.**,READ_SCHEMA,true,\n" @@ -964,22 +993,25 @@ public void testGrantAndGrantOpt() throws SQLException { // 2. USER1 has all privileges on root.** for (PrivilegeType item : PrivilegeType.values()) { + if (item.isRelationalPrivilege()) { + continue; + } String sql = "GRANT %s on root.** to USER user1"; - adminStmt.execute(String.format(sql, item.toString())); + adminStmt.execute(String.format(sql, item)); } // 3.admin lists privileges of user1 ResultSet resultSet = adminStmt.executeQuery("LIST PRIVILEGES OF USER user1"); String ans = - ",root.**,MANAGE_USER,false,\n" - + ",root.**,MANAGE_ROLE,false,\n" - + ",root.**,USE_TRIGGER,false,\n" - + ",root.**,USE_UDF,false,\n" - + ",root.**,USE_CQ,false,\n" - + ",root.**,USE_PIPE,false,\n" - + ",root.**,USE_MODEL,false,\n" - + ",root.**,EXTEND_TEMPLATE,false,\n" - + ",root.**,MANAGE_DATABASE,false,\n" - + ",root.**,MAINTAIN,false,\n" + ",,MANAGE_USER,false,\n" + + ",,MANAGE_ROLE,false,\n" + + ",,USE_TRIGGER,false,\n" + + ",,USE_UDF,false,\n" + + ",,USE_CQ,false,\n" + + ",,USE_PIPE,false,\n" + + ",,USE_MODEL,false,\n" + + ",,EXTEND_TEMPLATE,false,\n" + + ",,MANAGE_DATABASE,false,\n" + + ",,MAINTAIN,false,\n" + ",root.**,READ_DATA,false,\n" + ",root.**,WRITE_DATA,false,\n" + ",root.**,READ_SCHEMA,false,\n" @@ -988,21 +1020,24 @@ public void testGrantAndGrantOpt() throws SQLException { // 4. USER2 has all privilegs on root.** with grant option; for (PrivilegeType item : PrivilegeType.values()) { + if (item.isRelationalPrivilege()) { + continue; + } String sql = "GRANT %s on root.** to USER user2 with grant option"; - adminStmt.execute(String.format(sql, item.toString())); + adminStmt.execute(String.format(sql, item)); } resultSet = adminStmt.executeQuery("LIST PRIVILEGES OF USER user2"); ans = - ",root.**,MANAGE_USER,true,\n" - + ",root.**,MANAGE_ROLE,true,\n" - + ",root.**,USE_TRIGGER,true,\n" - + ",root.**,USE_UDF,true,\n" - + ",root.**,USE_CQ,true,\n" - + ",root.**,USE_PIPE,true,\n" - + ",root.**,USE_MODEL,true,\n" - + ",root.**,EXTEND_TEMPLATE,true,\n" - + ",root.**,MANAGE_DATABASE,true,\n" - + ",root.**,MAINTAIN,true,\n" + ",,MANAGE_USER,true,\n" + + ",,MANAGE_ROLE,true,\n" + + ",,USE_TRIGGER,true,\n" + + ",,USE_UDF,true,\n" + + ",,USE_CQ,true,\n" + + ",,USE_PIPE,true,\n" + + ",,USE_MODEL,true,\n" + + ",,EXTEND_TEMPLATE,true,\n" + + ",,MANAGE_DATABASE,true,\n" + + ",,MAINTAIN,true,\n" + ",root.**,READ_DATA,true,\n" + ",root.**,WRITE_DATA,true,\n" + ",root.**,READ_SCHEMA,true,\n" @@ -1019,16 +1054,16 @@ public void testGrantAndGrantOpt() throws SQLException { try { resultSet = userStmt.executeQuery("LIST PRIVILEGES OF USER user1"); ans = - ",root.**,MANAGE_USER,false,\n" - + ",root.**,MANAGE_ROLE,false,\n" - + ",root.**,USE_TRIGGER,false,\n" - + ",root.**,USE_UDF,false,\n" - + ",root.**,USE_CQ,false,\n" - + ",root.**,USE_PIPE,false,\n" - + ",root.**,USE_MODEL,false,\n" - + ",root.**,EXTEND_TEMPLATE,false,\n" - + ",root.**,MANAGE_DATABASE,false,\n" - + ",root.**,MAINTAIN,false,\n" + ",,MANAGE_USER,false,\n" + + ",,MANAGE_ROLE,false,\n" + + ",,USE_TRIGGER,false,\n" + + ",,USE_UDF,false,\n" + + ",,USE_CQ,false,\n" + + ",,USE_PIPE,false,\n" + + ",,USE_MODEL,false,\n" + + ",,EXTEND_TEMPLATE,false,\n" + + ",,MANAGE_DATABASE,false,\n" + + ",,MAINTAIN,false,\n" + ",root.**,READ_DATA,false,\n" + ",root.**,WRITE_DATA,false,\n" + ",root.**,READ_SCHEMA,false,\n" @@ -1052,21 +1087,21 @@ public void testGrantAndGrantOpt() throws SQLException { validateResultSet(resultSet, ans); userStmt.execute("GRANT MANAGE_ROLE ON root.** TO USER user3"); resultSet = userStmt.executeQuery("LIST PRIVILEGES OF USER user3"); - ans = ",root.**,MANAGE_ROLE,false,\n"; + ans = ",,MANAGE_ROLE,false,\n"; validateResultSet(resultSet, ans); userStmt.execute("REVOKE MANAGE_ROLE ON root.** FROM USER user1"); resultSet = userStmt.executeQuery("LIST PRIVILEGES OF USER user1"); ans = - ",root.**,MANAGE_USER,false,\n" - + ",root.**,USE_TRIGGER,false,\n" - + ",root.**,USE_UDF,false,\n" - + ",root.**,USE_CQ,false,\n" - + ",root.**,USE_PIPE,false,\n" - + ",root.**,USE_MODEL,false,\n" - + ",root.**,EXTEND_TEMPLATE,false,\n" - + ",root.**,MANAGE_DATABASE,false,\n" - + ",root.**,MAINTAIN,false,\n" + ",,MANAGE_USER,false,\n" + + ",,USE_TRIGGER,false,\n" + + ",,USE_UDF,false,\n" + + ",,USE_CQ,false,\n" + + ",,USE_PIPE,false,\n" + + ",,USE_MODEL,false,\n" + + ",,EXTEND_TEMPLATE,false,\n" + + ",,MANAGE_DATABASE,false,\n" + + ",,MAINTAIN,false,\n" + ",root.**,READ_DATA,false,\n" + ",root.**,WRITE_DATA,false,\n" + ",root.**,READ_SCHEMA,false,\n" @@ -1115,9 +1150,12 @@ public void testRevokeAndGrantOpt() throws SQLException { // user2 has all privileges without grant option on root.** // user2 has all privileges without grant option on root.t1.** for (PrivilegeType item : PrivilegeType.values()) { + if (item.isRelationalPrivilege()) { + continue; + } String sql = "GRANT %s on root.** to USER user1 WITH GRANT OPTION"; adminStmt.execute(String.format(sql, item)); - if (item.isPathRelevant()) { + if (item.isPathPrivilege()) { adminStmt.execute(String.format("GRANT %s on root.t1.** TO USER user2", item)); } sql = "GRANT %s on root.** to USER user2"; @@ -1130,6 +1168,9 @@ public void testRevokeAndGrantOpt() throws SQLException { try { // revoke privileges on root.** and root.t1.** for (PrivilegeType item : PrivilegeType.values()) { + if (item.isRelationalPrivilege()) { + continue; + } user1Stmt.execute(String.format("REVOKE %s ON root.** FROM USER user2", item)); } @@ -1244,4 +1285,142 @@ public void testCreateRoleIdentifierName() throws SQLException { adminStmt.execute("create role tail"); adminStmt.execute("create user tail 'password'"); } + + @Test + public void testClusterManagementSqlOfTreeModel() throws Exception { + ImmutableList clusterManagementSQLList = + ImmutableList.of( + // show cluster, nodes, regions, + "show ainodes", + "show confignodes", + "show datanodes", + "show cluster", + "show clusterid", + "show regions", + "show data regionid where database=root.**", + + // remove node + "remove datanode 0", + "remove confignode 0", + + // region operation + "migrate region 0 from 1 to 2", + "reconstruct region 0 on 1", + "extend region 0 to 1", + "remove region 0 from 1", + + // others + "show timeslotid where database=root.test", + "count timeslotid where database=root.test", + "show data seriesslotid where database=root.test", + "verify connection"); + + try (Connection adminCon = EnvFactory.getEnv().getConnection(); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("CREATE USER Jack 'temppw'"); + + try (Connection JackConnection = EnvFactory.getEnv().getConnection("Jack", "temppw"); + Statement Jack = JackConnection.createStatement()) { + testClusterManagementSqlImpl( + clusterManagementSQLList, + () -> adminStmt.execute("GRANT MAINTAIN ON root.** TO USER Jack"), + Jack); + } + } + } + + @Test + public void testClusterManagementSqlOfTableModel() throws Exception { + ImmutableList clusterManagementSQLList = + ImmutableList.of( + // show cluster, nodes, regions, + "show ainodes", + "show confignodes", + "show datanodes", + "show cluster", + "show clusterid", + "show regions", + + // remove node + "remove datanode 0", + "remove confignode 0", + + // region operation + "migrate region 0 from 1 to 2", + "reconstruct region 0 on 1", + "extend region 0 to 1", + "remove region 0 from 1"); + + try (Connection adminCon = EnvFactory.getEnv().getTableConnection(); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("CREATE USER Jack 'temppw'"); + + try (Connection JackConnection = + EnvFactory.getEnv().getConnection("Jack", "temppw", BaseEnv.TABLE_SQL_DIALECT); + Statement Jack = JackConnection.createStatement()) { + // Jack has no authority to execute these SQLs + for (String sql : clusterManagementSQLList) { + try { + Jack.execute(sql); + } catch (IoTDBSQLException e) { + if (TSStatusCode.NO_PERMISSION.getStatusCode() != e.getErrorCode()) { + fail( + String.format( + "SQL should fail because of no permission, but the error code is %d: %s", + e.getErrorCode(), sql)); + } + continue; + } + fail(String.format("SQL should fail because of no permission: %s", sql)); + } + } + } + } + + private void testClusterManagementSqlImpl( + List clusterManagementSqlList, Callable giveJackAuthority, Statement Jack) + throws Exception { + // Jack has no authority to execute these SQLs + for (String sql : clusterManagementSqlList) { + try { + Jack.execute(sql); + } catch (IoTDBSQLException e) { + if (TSStatusCode.NO_PERMISSION.getStatusCode() != e.getErrorCode()) { + fail( + String.format( + "SQL should fail because of no permission, but the error code is %d: %s", + e.getErrorCode(), sql)); + } + continue; + } + fail(String.format("SQL should fail because of no permission: %s", sql)); + } + + // Give Jack authority + giveJackAuthority.call(); + + // Jack is able to execute these SQLs now + for (String sql : clusterManagementSqlList) { + try { + // No exception is fine + Jack.execute(sql); + } catch (IoTDBSQLException e) { + // If there is an exception, error code must not be NO_PERMISSION + if (TSStatusCode.NO_PERMISSION.getStatusCode() == e.getErrorCode()) { + fail(String.format("SQL should not fail with no permission: %s", sql)); + } + } + } + } + + @Test + public void noNeedPrivilegeTest() { + createUser("tempuser", "temppw"); + String[] expectedHeader = new String[] {"CurrentUser"}; + String[] retArray = + new String[] { + "tempuser,", + }; + resultSetEqualTest("show current_user", expectedHeader, retArray, "tempuser", "temppw"); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityIT.java index ef9c147a742b9..6c13036b1e941 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityIT.java @@ -20,16 +20,17 @@ package org.apache.iotdb.db.it.auth; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.PrivilegeModelType; import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.utils.AuthUtils; import org.apache.iotdb.confignode.rpc.thrift.IConfigNodeRPCService; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.plan.statement.AuthorType; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -60,8 +61,8 @@ public class IoTDBClusterAuthorityIT { @Before public void setUp() throws Exception { - // Init 1C0D environment - EnvFactory.getEnv().initClusterEnvironment(1, 0); + // Init 1C1D environment + EnvFactory.getEnv().initClusterEnvironment(1, 1); } @After @@ -159,9 +160,11 @@ public void permissionTest() throws IllegalPathException { // check user privileges checkUserPrivilegesReq = new TCheckUserPrivilegesReq( - "tempuser0", - AuthUtils.serializePartialPathList(paths), - PrivilegeType.MANAGE_USER.ordinal()); + "tempuser0", + PrivilegeModelType.TREE.ordinal(), + PrivilegeType.MANAGE_USER.ordinal(), + false) + .setPaths(AuthUtils.serializePartialPathList(paths)); status = client.checkUserPrivileges(checkUserPrivilegesReq).getStatus(); assertEquals(TSStatusCode.NO_PERMISSION.getStatusCode(), status.getCode()); @@ -282,9 +285,11 @@ public void permissionTest() throws IllegalPathException { // check user privileges checkUserPrivilegesReq = new TCheckUserPrivilegesReq( - "tempuser0", - AuthUtils.serializePartialPathList(paths), - PrivilegeType.READ_DATA.ordinal()); + "tempuser0", + PrivilegeModelType.TREE.ordinal(), + PrivilegeType.READ_DATA.ordinal(), + false) + .setPaths(AuthUtils.serializePartialPathList(nodeNameList)); status = client.checkUserPrivileges(checkUserPrivilegesReq).getStatus(); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -367,10 +372,13 @@ public void permissionTest() throws IllegalPathException { status = authorizerResp.getStatus(); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); assertEquals(ColumnHeaderConstant.PRIVILEGES, authorizerResp.getTag()); - assertEquals("tempuser0", authorizerResp.getPermissionInfo().getUserInfo().getUsername()); assertEquals( - new ArrayList<>(), authorizerResp.getPermissionInfo().getUserInfo().getPrivilegeList()); - assertEquals(1, authorizerResp.getPermissionInfo().getUserInfo().getRoleListSize()); + "tempuser0", + authorizerResp.getPermissionInfo().getUserInfo().getPermissionInfo().getName()); + assertEquals( + new ArrayList<>(), + authorizerResp.getPermissionInfo().getUserInfo().getPermissionInfo().getPrivilegeList()); + assertEquals(1, authorizerResp.getPermissionInfo().getUserInfo().getRoleSet().size()); // list privileges role authorizerReq = @@ -443,17 +451,32 @@ public void permissionTest() throws IllegalPathException { assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); assertNull(authorizerResp.getMemberInfo()); assertEquals(new HashMap<>(), authorizerResp.getPermissionInfo().getRoleInfo()); + assertEquals(new HashSet<>(), authorizerResp.getPermissionInfo().getUserInfo().getRoleSet()); assertEquals( - new ArrayList<>(), authorizerResp.getPermissionInfo().getUserInfo().getRoleList()); - assertEquals( - PrivilegeType.getPathPriCount(), - authorizerResp.getPermissionInfo().getUserInfo().getPrivilegeList().get(0).priSet.size()); + PrivilegeType.getPrivilegeCount(PrivilegeModelType.TREE), + authorizerResp + .getPermissionInfo() + .getUserInfo() + .getPermissionInfo() + .getPrivilegeList() + .get(0) + .priSet + .size()); assertEquals( - PrivilegeType.getSysPriCount(), - authorizerResp.getPermissionInfo().getUserInfo().getSysPriSet().size()); + PrivilegeType.getPrivilegeCount(PrivilegeModelType.SYSTEM), + authorizerResp + .getPermissionInfo() + .getUserInfo() + .getPermissionInfo() + .getSysPriSet() + .size()); assertEquals( - PrivilegeType.getSysPriCount(), - authorizerResp.getPermissionInfo().getUserInfo().getSysPriSetGrantOptSize()); + PrivilegeType.getPrivilegeCount(PrivilegeModelType.SYSTEM), + authorizerResp + .getPermissionInfo() + .getUserInfo() + .getPermissionInfo() + .getSysPriSetGrantOptSize()); authorizerReq = new TAuthorizerReq( @@ -473,8 +496,9 @@ public void permissionTest() throws IllegalPathException { checkUserPrivilegesReq = new TCheckUserPrivilegesReq( "tempuser0", - AuthUtils.serializePartialPathList(new ArrayList<>()), - PrivilegeType.MANAGE_USER.ordinal()); + PrivilegeModelType.SYSTEM.ordinal(), + PrivilegeType.MANAGE_USER.ordinal(), + false); status = client.checkUserPrivileges(checkUserPrivilegesReq).getStatus(); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -496,8 +520,9 @@ public void permissionTest() throws IllegalPathException { checkUserPrivilegesReq = new TCheckUserPrivilegesReq( "tempuser0", - AuthUtils.serializePartialPathList(new ArrayList<>()), - PrivilegeType.MANAGE_DATABASE.ordinal()); + PrivilegeModelType.SYSTEM.ordinal(), + PrivilegeType.MANAGE_DATABASE.ordinal(), + false); status = client.checkUserPrivileges(checkUserPrivilegesReq).getStatus(); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityRelationalIT.java new file mode 100644 index 0000000000000..7d1eec7e72caf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBClusterAuthorityRelationalIT.java @@ -0,0 +1,512 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.auth; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.PrivilegeModelType; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.utils.AuthUtils; +import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; +import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; +import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; +import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; +import org.apache.iotdb.confignode.rpc.thrift.TLoginReq; +import org.apache.iotdb.confignode.rpc.thrift.TPermissionInfoResp; +import org.apache.iotdb.confignode.rpc.thrift.TRoleResp; +import org.apache.iotdb.confignode.rpc.thrift.TUserResp; +import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class, TableClusterIT.class}) +public class IoTDBClusterAuthorityRelationalIT { + @Before + public void setUp() throws Exception { + // Init 1C1D environment + EnvFactory.getEnv().initClusterEnvironment(1, 1); + } + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private void runAndCheck(SyncConfigNodeIServiceClient client, TAuthorizerRelationalReq req) + throws TException { + TSStatus status; + status = client.operateRPermission(req); + assertEquals(status.getCode(), TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + private void cleanUserAndRole(SyncConfigNodeIServiceClient client) throws TException { + TSStatus status; + + // clean user + TAuthorizerRelationalReq authorizerReq = + new TAuthorizerRelationalReq( + AuthorRType.LIST_USER.ordinal(), "", "", "", "", "", Collections.emptySet(), false); + + TAuthorizerResp authorizerResp = client.queryRPermission(authorizerReq); + status = authorizerResp.getStatus(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List allUsers = authorizerResp.getMemberInfo(); + for (String user : allUsers) { + if (!user.equals("root")) { + authorizerReq = + new TAuthorizerRelationalReq( + AuthorRType.DROP_USER.ordinal(), + user, + "", + "", + "", + "", + Collections.emptySet(), + false); + status = client.operateRPermission(authorizerReq); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + } + + // clean role + authorizerReq = + new TAuthorizerRelationalReq( + AuthorRType.LIST_ROLE.ordinal(), "", "", "", "", "", Collections.emptySet(), false); + + authorizerResp = client.queryRPermission(authorizerReq); + status = authorizerResp.getStatus(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List allRoles = authorizerResp.getMemberInfo(); + for (String role : allRoles) { + authorizerReq = + new TAuthorizerRelationalReq( + AuthorRType.DROP_ROLE.ordinal(), role, "", "", "", "", Collections.emptySet(), false); + status = client.operateRPermission(authorizerReq); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + } + + private void createUserORRoleAndCheck( + SyncConfigNodeIServiceClient client, String name, boolean isUser, String password) + throws TException { + TSStatus status; + TAuthorizerRelationalReq authorizerReq = + new TAuthorizerRelationalReq( + isUser ? AuthorRType.CREATE_USER.ordinal() : AuthorRType.CREATE_ROLE.ordinal(), + isUser ? name : "", + isUser ? "" : name, + password, + "", + "", + Collections.emptySet(), + false); + status = client.operateRPermission(authorizerReq); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + authorizerReq = + new TAuthorizerRelationalReq( + isUser ? AuthorRType.LIST_USER.ordinal() : AuthorRType.LIST_ROLE.ordinal(), + "", + "", + "", + "", + "", + Collections.emptySet(), + false); + TAuthorizerResp resp = client.queryRPermission(authorizerReq); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); + assertTrue(resp.getMemberInfo().contains(name)); + } + + private void grantSysPrivilegeAndCheck( + SyncConfigNodeIServiceClient client, + String userName, + String roleName, + boolean toUser, + PrivilegeType sysPriv, + boolean grantOpt) + throws TException { + TSStatus status; + TAuthorizerRelationalReq authorizerRelationalReq = + new TAuthorizerRelationalReq( + toUser ? AuthorRType.GRANT_USER_SYS.ordinal() : AuthorRType.GRANT_ROLE_SYS.ordinal(), + userName, + roleName, + "", + "", + "", + Collections.singleton(sysPriv.ordinal()), + grantOpt); + + status = client.operateRPermission(authorizerRelationalReq); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + TCheckUserPrivilegesReq checkUserPrivilegesReq = + new TCheckUserPrivilegesReq( + userName, PrivilegeModelType.SYSTEM.ordinal(), sysPriv.ordinal(), grantOpt); + TPermissionInfoResp resp = client.checkUserPrivileges(checkUserPrivilegesReq); + assertEquals(resp.status.getCode(), TSStatusCode.SUCCESS_STATUS.getStatusCode()); + assertTrue( + toUser + ? resp.getUserInfo().getPermissionInfo().getSysPriSet().contains(sysPriv.ordinal()) + : resp.getRoleInfo().containsKey(roleName) + && resp.getRoleInfo().get(roleName).getSysPriSet().contains(sysPriv.ordinal())); + if (grantOpt) { + assertTrue( + toUser + ? resp.getUserInfo() + .getPermissionInfo() + .getSysPriSetGrantOpt() + .contains(sysPriv.ordinal()) + : resp.getRoleInfo().containsKey(roleName) + && resp.getRoleInfo() + .get(roleName) + .getSysPriSetGrantOpt() + .contains(sysPriv.ordinal())); + } + } + + private void grantPrivilegeAndCheck( + SyncConfigNodeIServiceClient client, + String username, + String rolename, + boolean toUser, + PrivilegeUnion union) + throws TException { + TSStatus status; + AuthorRType type; + if (union.isForAny()) { + type = toUser ? AuthorRType.GRANT_USER_ANY : AuthorRType.GRANT_ROLE_ANY; + } else if (union.getTbName() == null) { + type = toUser ? AuthorRType.GRANT_USER_DB : AuthorRType.GRANT_ROLE_DB; + } else { + type = toUser ? AuthorRType.GRANT_USER_TB : AuthorRType.GRANT_ROLE_TB; + } + TAuthorizerRelationalReq authorizerRelationalReq = + new TAuthorizerRelationalReq( + type.ordinal(), + toUser ? username : "", + toUser ? "" : rolename, + "", + union.getDBName() == null ? "" : union.getDBName(), + union.getTbName() == null ? "" : union.getTbName(), + Collections.singleton(union.getPrivilegeType().ordinal()), + union.isGrantOption()); + status = client.operateRPermission(authorizerRelationalReq); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + int reqtype = -1; + if (union.getPrivilegeType().isRelationalPrivilege()) { + reqtype = PrivilegeModelType.RELATIONAL.ordinal(); + } else if (union.getPrivilegeType().isSystemPrivilege()) { + reqtype = PrivilegeModelType.SYSTEM.ordinal(); + } + TCheckUserPrivilegesReq checkUserPrivilegesReq = + new TCheckUserPrivilegesReq( + username, reqtype, union.getPrivilegeType().ordinal(), union.isGrantOption()); + if (union.getDBName() != null) { + checkUserPrivilegesReq.setDatabase(union.getDBName()); + } + if (union.getTbName() != null) { + checkUserPrivilegesReq.setTable(union.getTbName()); + } + + TPermissionInfoResp resp = client.checkUserPrivileges(checkUserPrivilegesReq); + assertEquals(resp.status.getCode(), TSStatusCode.SUCCESS_STATUS.getStatusCode()); + if (toUser) { + TUserResp userInfo = resp.getUserInfo(); + if (union.isForAny()) { + assertTrue( + userInfo + .getPermissionInfo() + .getAnyScopeGrantSet() + .contains(union.getPrivilegeType().ordinal())); + } else if (union.getTbName() == null) { + assertTrue(userInfo.getPermissionInfo().getDbPrivilegeMap().containsKey(union.getDBName())); + assertTrue( + userInfo + .getPermissionInfo() + .getDbPrivilegeMap() + .get(union.getDBName()) + .getPrivileges() + .contains(union.getPrivilegeType().ordinal())); + } else { + assertTrue(userInfo.getPermissionInfo().getDbPrivilegeMap().containsKey(union.getDBName())); + assertTrue( + userInfo + .getPermissionInfo() + .getDbPrivilegeMap() + .get(union.getDBName()) + .getTablePrivilegeMap() + .containsKey(union.getTbName())); + assertTrue( + userInfo + .getPermissionInfo() + .getDbPrivilegeMap() + .get(union.getDBName()) + .getTablePrivilegeMap() + .get(union.getTbName()) + .getPrivileges() + .contains(union.getPrivilegeType().ordinal())); + } + } else { + assertTrue(resp.getRoleInfo().containsKey(rolename)); + TRoleResp roleResp = resp.getRoleInfo().get(rolename); + if (union.isForAny()) { + assertTrue(roleResp.getAnyScopeGrantSet().contains(union.getPrivilegeType().ordinal())); + } + if (union.getTbName() == null) { + assertTrue(roleResp.getDbPrivilegeMap().containsKey(union.getDBName())); + assertTrue( + roleResp + .getDbPrivilegeMap() + .get(union.getDBName()) + .getPrivileges() + .contains(union.getPrivilegeType().ordinal())); + } else { + assertTrue(roleResp.getDbPrivilegeMap().containsKey(union.getDBName())); + assertTrue( + roleResp + .getDbPrivilegeMap() + .get(union.getDBName()) + .getTablePrivilegeMap() + .containsKey(union.getTbName())); + assertTrue( + roleResp + .getDbPrivilegeMap() + .get(union.getDBName()) + .getTablePrivilegeMap() + .get(union.getTbName()) + .getPrivileges() + .contains(union.getPrivilegeType().ordinal())); + } + } + } + + private void expectSuccess(TPermissionInfoResp resp) { + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); + } + + private void expectFailed(TPermissionInfoResp resp) { + assertEquals(TSStatusCode.NO_PERMISSION.getStatusCode(), resp.getStatus().getCode()); + } + + @Test + public void permissionTest() + throws TException, ClientManagerException, IOException, InterruptedException { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + cleanUserAndRole(client); + createUserORRoleAndCheck(client, "user1", true, "password"); + createUserORRoleAndCheck(client, "role1", false, ""); + runAndCheck( + client, + new TAuthorizerRelationalReq( + AuthorRType.GRANT_USER_ROLE.ordinal(), + "user1", + "role1", + "", + "", + "", + Collections.emptySet(), + false)); + grantSysPrivilegeAndCheck(client, "user1", "role1", true, PrivilegeType.MANAGE_USER, false); + grantSysPrivilegeAndCheck(client, "user1", "role1", true, PrivilegeType.MANAGE_ROLE, true); + grantPrivilegeAndCheck( + client, "user1", "", true, new PrivilegeUnion("database", "table", PrivilegeType.SELECT)); + grantPrivilegeAndCheck( + client, "user1", "", true, new PrivilegeUnion("database2", PrivilegeType.ALTER)); + grantPrivilegeAndCheck( + client, "user1", "", true, new PrivilegeUnion(PrivilegeType.INSERT, true, true)); + grantPrivilegeAndCheck( + client, + "user1", + "role1", + false, + new PrivilegeUnion("database", "table2", PrivilegeType.DELETE)); + grantPrivilegeAndCheck( + client, + "user1", + "role1", + false, + new PrivilegeUnion("database2", PrivilegeType.CREATE, true)); + + // privileges status + // user1 <-- role1 + // user1 : MANAGE_USER, MANAGE_ROLE(with grant option) + // "database"."table" select; + // "database2".* alter; + // any insert(with grant option); + // role1: MAINTAIN(with grant option) + // "database"."table2" delete, + // "database2".* create(with grant option); + + // check login + TLoginReq req = new TLoginReq("user1", "password"); + expectSuccess(client.login(req)); + + // check user has role. + TAuthorizerReq user_role_req = + new TAuthorizerReq( + 0, + "user1", + "role1", + "", + "", + Collections.emptySet(), + false, + AuthUtils.serializePartialPathList(Collections.emptyList())); + expectSuccess(client.checkRoleOfUser(user_role_req)); + + // check db is visible + TCheckUserPrivilegesReq check_req = + new TCheckUserPrivilegesReq("user1", PrivilegeModelType.RELATIONAL.ordinal(), -1, false) + .setDatabase("database"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check db is visible, because any privileges, so database3 is visible. + check_req = + new TCheckUserPrivilegesReq("user1", PrivilegeModelType.RELATIONAL.ordinal(), -1, false) + .setDatabase("database3"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check table is visible. + check_req = + new TCheckUserPrivilegesReq("user1", PrivilegeModelType.RELATIONAL.ordinal(), -1, false) + .setDatabase("database") + .setTable("table"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check db privileges + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.ALTER.ordinal(), + false) + .setDatabase("database2"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check db privileges, success for any + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.INSERT.ordinal(), + false) + .setDatabase("database10"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check db privileges, success for role + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.CREATE.ordinal(), + false) + .setDatabase("database2"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check db privileges grant option, + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.CREATE.ordinal(), + true) + .setDatabase("database2"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check tb privilege + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.SELECT.ordinal(), + false) + .setDatabase("database") + .setTable("table"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check tb privilege + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.ALTER.ordinal(), + false) + .setDatabase("database2") + .setTable("table"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check tb privilege + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.CREATE.ordinal(), + false) + .setDatabase("database2") + .setTable("table"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check tb privilege + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.DELETE.ordinal(), + false) + .setDatabase("database") + .setTable("table2"); + expectSuccess(client.checkUserPrivileges(check_req)); + + // check tb privilege + check_req = + new TCheckUserPrivilegesReq( + "user1", + PrivilegeModelType.RELATIONAL.ordinal(), + PrivilegeType.SELECT.ordinal(), + false) + .setDatabase("database") + .setTable("table2"); + expectFailed(client.checkUserPrivileges(check_req)); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java new file mode 100644 index 0000000000000..91efa12273c3a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java @@ -0,0 +1,618 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.auth; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.jdbc.IoTDBSQLException; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRelationalAuthIT { + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void listUserPrivileges() throws SQLException { + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + + adminStmt.execute("create user testuser 'password'"); + adminStmt.execute("create database testdb"); + adminStmt.execute("GRANT MANAGE_USER to user testuser"); + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT MANAGE_DATABASE to user testuser"); + }); + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT read_data on database db1 to user testuser"); + }); + // No Maintain privilege in table model. + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT MAINTAIN to user testuser"); + }); + + adminStmt.execute("GRANT MANAGE_ROLE TO USER testuser"); + adminStmt.execute("GRANT SELECT ON ANY TO USER testuser"); + adminStmt.execute("GRANT INSERT ON ANY TO USER testuser"); + adminStmt.execute("GRANT DELETE ON ANY TO USER testuser"); + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT SELECT ON TABLE TB to user testuser"); + }); + adminStmt.execute("GRANT SELECT ON DATABASE TESTDB TO USER testuser"); + adminStmt.execute("GRANT SELECT ON DATABASE TESTDB TO USER testuser WITH GRANT OPTION"); + + adminStmt.execute("use testdb"); + adminStmt.execute("GRANT SELECT ON TABLE TB to user testuser"); + adminStmt.execute("GRANT INSERT ON TABLE TB to user testuser with grant option"); + adminStmt.execute("GRANT DROP ON testdb.tb to user testuser with grant option"); + + ResultSet rs = adminStmt.executeQuery("LIST PRIVILEGES OF USER testuser"); + Set ans = + new HashSet<>( + Arrays.asList( + ",,MANAGE_USER,false,", + ",,MANAGE_ROLE,false,", + ",*.*,SELECT,false,", + ",*.*,INSERT,false,", + ",*.*,DELETE,false,", + ",testdb.*,SELECT,true,", + ",testdb.tb,SELECT,false,", + ",testdb.tb,INSERT,true,", + ",testdb.tb,DROP,true,")); + TestUtils.assertResultSetEqual(rs, "Role,Scope,Privileges,GrantOption,", ans); + adminStmt.execute("create role testrole"); + adminStmt.execute("GRANT ROLE testrole to testuser"); + rs = adminStmt.executeQuery("LIST USER OF ROLE testrole"); + TestUtils.assertResultSetEqual(rs, "User,", Collections.singleton("testuser,")); + rs = adminStmt.executeQuery("LIST ROLE OF USER testuser"); + TestUtils.assertResultSetEqual(rs, "Role,", Collections.singleton("testrole,")); + } + } + + @Test + public void checkAuthorStatementPrivilegeCheck() throws SQLException { + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user testuser 'password'"); + adminStmt.execute("create user testuser2 'password'"); + adminStmt.execute("create role testrole"); + adminStmt.execute("create database testdb"); + + // cannot create admin user + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("CREATE USER root 'password'"); + }); + // cannot create admin role + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("CREATE role root"); + }); + + // cannot grant role to admin user + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT ROLE testrole to root"); + }); + + // cannot grant privilege to admin user + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT MANAGE_USER to root"); + }); + + // admin can do all things below. + adminStmt.execute("GRANT MANAGE_USER to user testuser2 with grant option"); + adminStmt.execute("GRANT MANAGE_ROLE to user testuser"); + + adminStmt.execute("use testdb"); + adminStmt.execute("GRANT SELECT ON TABLE TB to user testuser"); + adminStmt.execute("GRANT INSERT ON TABLE TB to user testuser"); + adminStmt.execute("GRANT INSERT ON DATABASE testdb to user testuser"); + adminStmt.execute("GRANT ALTER ON ANY to user testuser"); + + adminStmt.execute("GRANT DROP ON TABLE TB to user testuser2 with grant option"); + adminStmt.execute("GRANT CREATE ON TABLE TB to user testuser2 with grant option"); + adminStmt.execute("GRANT DROP ON DATABASE testdb to user testuser2 with grant option"); + adminStmt.execute("GRANT SELECT ON ANY to user testuser2 with grant option"); + + adminStmt.execute("GRANT ROLE testrole to testuser"); + } + + try (Connection userCon1 = + EnvFactory.getEnv().getConnection("testuser", "password", BaseEnv.TABLE_SQL_DIALECT); + Statement userStmt = userCon1.createStatement()) { + // 1. user1's privileges + // testdb.TB select + // testdb.TB insert + // testdb.* insert + // any alter + // manage_role + + // cannot create user + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("CREATE USER testuser3 'password'"); + }); + // can create role + userStmt.execute("CREATE ROLE testrole2"); + // can grant role to user + userStmt.execute("GRANT ROLE testrole2 to testuser"); + // cannot grant privileges to other + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("GRANT SELECT ON testdb.TB to role testrole2"); + }); + + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("GRANT ALTER ON ANY to role testrole2"); + }); + + // cannot grant manage_role to other + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("GRANT manage_role to role testrole2"); + }); + + // can list itself privileges and the all roles privileges + ResultSet rs = userStmt.executeQuery("List privileges of user testuser"); + Set ans = + new HashSet<>( + Arrays.asList( + ",,MANAGE_ROLE,false,", + ",*.*,ALTER,false,", + ",testdb.*,INSERT,false,", + ",testdb.tb,SELECT,false,", + ",testdb.tb,INSERT,false,")); + TestUtils.assertResultSetEqual(rs, "Role,Scope,Privileges,GrantOption,", ans); + rs = userStmt.executeQuery("List privileges of role testrole"); + TestUtils.assertResultSetEqual( + rs, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); + rs = userStmt.executeQuery("List privileges of role testrole2"); + TestUtils.assertResultSetEqual( + rs, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); + // testdb.TB's privilege is not grant option. + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("GRANT insert on testdb.TB to role testrole2"); + }); + + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("GRANT ALTER on any to role testrole2"); + }); + } + + try (Connection userCon1 = + EnvFactory.getEnv().getConnection("testuser2", "password", BaseEnv.TABLE_SQL_DIALECT); + Statement userStmt = userCon1.createStatement()) { + // 2. user2's privileges + // MANAGE_USER with grant option + // testdb.tb drop with grant option + // testdb.tb create with grant option + // testdb.* drop with grant option + // any select with grant option + + // can create user. + userStmt.execute("CREATE USER testuser3 'password'"); + + // can not create role + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("CREATE ROLE testrole3"); + }); + + // cannot list role's privileges + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.executeQuery("List privileges of role testrole"); + }); + + userStmt.execute("GRANT drop on database testdb to user testuser3"); + userStmt.execute("GRANT SELECT ON database testdb to user testuser3"); + ResultSet rs = userStmt.executeQuery("List privileges of user testuser3"); + Set ans = + new HashSet<>(Arrays.asList(",testdb.*,SELECT,false,", ",testdb.*,DROP,false,")); + TestUtils.assertResultSetEqual(rs, "Role,Scope,Privileges,GrantOption,", ans); + userStmt.execute("REVOKE SELECT ON DATABASE testdb from user testuser3"); + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("GRANT CREATE ON DATABASE testdb to user testuser3"); + }); + + rs = userStmt.executeQuery("List privileges of user testuser3"); + TestUtils.assertResultSetEqual( + rs, "Role,Scope,Privileges,GrantOption,", Collections.singleton(",testdb.*,DROP,false,")); + } + } + + @Test + public void checkGrantRevokeAllPrivileges() throws SQLException { + // In this IT: + // grant + // 1. grant all on table tb1 with grant option + // 2. grant all on database testdb + // 3. grant all on any + // revoke + // 1. revoke grant option for all on table tb1 + // 2. revoke all on table tb1 + // 3. revoke all on database testdb + // 4. revoke all on any + // grant and revoke + // 1. grant all on user/role + // 2. revoke all on any + // 3. revoke all on user/role + + for (boolean isUser : new boolean[] {true, false}) { + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create database testdb"); + adminStmt.execute(isUser ? "create user test 'password'" : "create role test"); + adminStmt.execute("use testdb"); + + // 1. grant all on table tb1 with grant option + adminStmt.execute( + "grant all on table tb1 to " + + (isUser ? "user test" : "role test") + + " with grant option"); + Set listPrivilegeResult = new HashSet<>(); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",true,"); + } + } + ResultSet resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 2. grant all on database testdb + adminStmt.execute( + "grant all on database testdb to " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add( + (isUser ? "," : "test,") + "testdb.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 3. grant all on any + adminStmt.execute("grant all on any to " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add((isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 1. revoke grant option for all on table tb1 + adminStmt.execute( + "revoke grant option for all on table tb1 from " + + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",true,"); + listPrivilegeResult.add( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 2. revoke all on table tb1 + adminStmt.execute("revoke all on table tb1 from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 3. revoke all on database testdb + adminStmt.execute( + "revoke all on database testdb from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "testdb.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 4. revoke all on any + adminStmt.execute("revoke all on any from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } + } + Assert.assertTrue(listPrivilegeResult.isEmpty()); + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 1. grant all on user/role + adminStmt.execute("grant all to " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add((isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } else if (privilegeType.forRelationalSys()) { + listPrivilegeResult.add((isUser ? "," : "test,") + "," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 2. revoke all on any + adminStmt.execute("revoke all on any from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 3. revoke all on user/role + adminStmt.execute("revoke all from " + (isUser ? "user test" : "role test")); + listPrivilegeResult.clear(); + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + adminStmt.execute("drop database testdb"); + adminStmt.execute(isUser ? "drop user test" : "drop role test"); + } + } + + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + adminStmt.execute("create user test2 'password'"); + adminStmt.execute("grant all to user test"); + adminStmt.execute("grant all to user test2 with grant option"); + adminStmt.execute("revoke SELECT ON ANY from user test"); + adminStmt.execute("create role role1"); + adminStmt.execute("grant all to role role1 with grant option"); + } + + Set listUserPrivilegeResult = new HashSet<>(); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType == PrivilegeType.SELECT) { + continue; + } + if (privilegeType.isRelationalPrivilege()) { + listUserPrivilegeResult.add(",*.*," + privilegeType + ",false,"); + } + if (privilegeType.forRelationalSys()) { + listUserPrivilegeResult.add(",," + privilegeType + ",false,"); + } + } + + Set listRolePrivilegeResult = new HashSet<>(); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listRolePrivilegeResult.add("role1,*.*," + privilegeType + ",true,"); + } + if (privilegeType.forRelationalSys()) { + listRolePrivilegeResult.add("role1,," + privilegeType + ",true,"); + } + } + + try (Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + Statement userConStatement = userCon.createStatement()) { + ResultSet resultSet = userConStatement.executeQuery("List privileges of user test"); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listUserPrivilegeResult); + + // Have manage_role privilege + resultSet = userConStatement.executeQuery("List privileges of role role1"); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listRolePrivilegeResult); + + // Do not have grant option + Assert.assertThrows( + SQLException.class, + () -> { + userConStatement.execute("GRANT SELECT ON DATABASE TEST to role role1"); + }); + + // Do not have grant option + Assert.assertThrows( + SQLException.class, + () -> { + userConStatement.execute("GRANT ALL to user test2"); + }); + } + + try (Connection userCon = + EnvFactory.getEnv().getConnection("test2", "password", BaseEnv.TABLE_SQL_DIALECT); + Statement userConStatement = userCon.createStatement()) { + // user2 can grant all to user test + userConStatement.execute("GRANT ALL to user test"); + // user2 can revoke all from user test + userConStatement.execute("REVOKE ALL from user test"); + + userConStatement.execute("GRANT ALL to user test"); + } + + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("revoke MANAGE_USER from user test2"); + } + + try (Connection userCon = + EnvFactory.getEnv().getConnection("test2", "password", BaseEnv.TABLE_SQL_DIALECT); + Statement userConStatement = userCon.createStatement()) { + // user2 can not grant all to user test + Assert.assertThrows( + SQLException.class, + () -> { + userConStatement.execute("GRANT ALL to user test2"); + }); + + // user2 can not revoke all from user test because does not hava all privileges + Assert.assertThrows( + SQLException.class, + () -> { + userConStatement.execute("REVOKE ALL to user test2"); + }); + } + + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("REVOKE ALL FROM USER test"); + ResultSet resultSet = adminStmt.executeQuery("List privileges of user test"); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); + adminStmt.execute("GRANT ALL ON db1.test TO USER test"); + adminStmt.execute("GRANT ALL ON DATABASE db2 TO USER test with grant option"); + resultSet = adminStmt.executeQuery("List privileges of user test"); + Set resultSetALL = new HashSet<>(); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + resultSetALL.add(",db2.*," + privilegeType + ",true,"); + resultSetALL.add(",db1.test," + privilegeType + ",false,"); + } + } + TestUtils.assertResultSetEqual(resultSet, "Role,Scope,Privileges,GrantOption,", resultSetALL); + adminStmt.execute("REVOKE ALL FROM USER test"); + resultSet = adminStmt.executeQuery("List privileges of user test"); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); + } + } + + @Test + public void testCreateUserAndRole() throws SQLException { + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + // normal case + adminStmt.execute("create user testuser 'password'"); + // username abnormal + adminStmt.execute("create user \"!@#$%^*()_+-=1\" 'password'"); + + // username and password abnormal + adminStmt.execute("create user \"!@#$%^*()_+-=2\" '!@#$%^*()_+-='"); + + // rolename abnormal + adminStmt.execute("create role \"!@#$%^*()_+-=3\" "); + + ResultSet resultSet = adminStmt.executeQuery("List user"); + Set resultSetList = new HashSet<>(); + resultSetList.add("root,"); + resultSetList.add("testuser,"); + resultSetList.add("!@#$%^*()_+-=1,"); + resultSetList.add("!@#$%^*()_+-=2,"); + TestUtils.assertResultSetEqual(resultSet, "User,", resultSetList); + resultSet = adminStmt.executeQuery("List role"); + TestUtils.assertResultSetEqual(resultSet, "Role,", Collections.singleton("!@#$%^*()_+-=3,")); + adminStmt.execute("GRANT role \"!@#$%^*()_+-=3\" to \"!@#$%^*()_+-=1\""); + adminStmt.execute("ALTER user \"!@#$%^*()_+-=1\" set password '!@#$%^*()_+-=\'"); + } + + try (Connection userCon = + EnvFactory.getEnv() + .getConnection("!@#$%^*()_+-=1", "!@#$%^*()_+-=", BaseEnv.TABLE_SQL_DIALECT); + Statement userConStatement = userCon.createStatement()) { + // List his role. + ResultSet set = userConStatement.executeQuery("List role of user \"!@#$%^*()_+-=1\""); + TestUtils.assertResultSetEqual(set, "Role,", Collections.singleton("!@#$%^*()_+-=3,")); + } catch (IoTDBSQLException e) { + Assert.fail(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java index eee4a86dc0845..822086c58cab6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.it.auth; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -32,22 +32,22 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.TIME; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.countDevicesColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.countNodesColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.countTimeSeriesColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showChildNodesColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showChildPathsColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDevicesColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showStorageGroupsColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTTLColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTimeSeriesColumnHeaders; import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.createUser; import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQuery; import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSeriesPrivilege; import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.TIME; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.countDevicesColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.countNodesColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.countTimeSeriesColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showChildNodesColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showChildPathsColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showDevicesColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showStorageGroupsColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showTTLColumnHeaders; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showTimeSeriesColumnHeaders; @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java index 4b3cebd68f1b3..c3ef62aa67e31 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java @@ -219,7 +219,7 @@ public void maintainOperationsTest() { executeNonQuery("show queries", "test", "test123"); assertNonQueryTestFail( "kill query 'test'", - "305: Please ensure your input is correct", + "701: Please ensure your input is correct", "test", "test123"); executeNonQuery("show cluster", "test", "test123"); @@ -247,7 +247,5 @@ public void adminOperationsTest() { "803: Only the admin user can perform this operation", "test", "test123"); - assertTestFail( - "show regions", "803: Only the admin user can perform this operation", "test", "test123"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBTemplatePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBTemplatePermissionIT.java index 7ac8a1e96a578..52f6e01fc69a3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBTemplatePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBTemplatePermissionIT.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.it.auth; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -32,13 +32,13 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showPathsUsingTemplateHeaders; import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.createUser; import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQuery; import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSeriesPrivilege; import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; import static org.apache.iotdb.db.it.utils.TestUtils.revokeUserSeriesPrivilege; -import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showPathsUsingTemplateHeaders; /** * This Class contains integration tests for SystemPermissions but {@link PrivilegeType#MANAGE_USER} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBDiffFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBDiffFunctionIT.java index 37887b4d382ab..970bbc2a745fc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBDiffFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBDiffFunctionIT.java @@ -38,7 +38,6 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBDiffFunctionIT { - // 2 devices 4 regions protected static final String[] SQLs = new String[] { "CREATE DATABASE root.db", diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBRoundFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBRoundFunctionIT.java index a34e069baa079..e16871b902291 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBRoundFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBRoundFunctionIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.builtinfunction.scalar; import org.apache.iotdb.it.env.EnvFactory; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBSubStringFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBSubStringFunctionIT.java index cf4d8bb84ffae..15e2d400497cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBSubStringFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBSubStringFunctionIT.java @@ -254,19 +254,19 @@ public void testRoundBooleanAndText() { // Negative characters length assertTestFail( "select SUBSTRING(s1,1,-10) from root.**", - TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Argument exception,the scalar function [SUBSTRING] beginPosition and length must be greater than 0"); // Negative characters begin assertTestFail( "select SUBSTRING(s1,-1,10) from root.**", - TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Argument exception,the scalar function [SUBSTRING] beginPosition and length must be greater than 0"); // Negative characters begin assertTestFail( "select SUBSTRING(s1 from -1 for 10) from root.**", - TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Argument exception,the scalar function [SUBSTRING] beginPosition and length must be greater than 0"); // Negative characters begin diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java index be8148758f6d9..399694b5e2cee 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java @@ -72,7 +72,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: Specifying time range in GROUP BY TIME clause is prohibited.", e.getMessage()); } @@ -92,7 +92,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: Specifying time filters in the query body is prohibited.", e.getMessage()); } @@ -128,7 +128,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: The query body misses an INTO clause.", e.getMessage()); } @@ -149,7 +149,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: Every interval [50] should not be lower than the `continuous_query_minimum_every_interval` [1000] configured.", e.getMessage()); } @@ -189,7 +189,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: The start time offset should be greater than 0.", e.getMessage()); } @@ -229,7 +229,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: The start time offset should be greater than end time offset.", e.getMessage()); } @@ -249,7 +249,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: The start time offset should be greater than end time offset.", e.getMessage()); } @@ -269,7 +269,7 @@ public void testCreateWrongCQ() { fail(); } catch (Exception e) { assertEquals( - TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode() + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": CQ: The start time offset should be greater than or equal to every interval.", e.getMessage()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java index 64f246d316bfd..321edef0f40eb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java @@ -88,9 +88,9 @@ public void testFill() { String[] resultSet = new String[] { - "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", - "2,1.2,2,2,null,null,", - "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + "1,1.1,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,1970-01-01T00:00:00.002Z,2,null,null,", + "3,1.3,1997-01-03T00:00:00.001Z,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", }; String expectedQueryHeader = @@ -100,9 +100,9 @@ public void testFill() { resultSet = new String[] { - "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "1,1.1,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", "2,1.2,null,1997-07-02,null,1997-07-02,", - "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + "3,1.3,1997-01-03T00:00:00.001Z,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", }; resultSetEqualTest( @@ -112,9 +112,9 @@ public void testFill() { resultSet = new String[] { - "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "1,1.1,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", "2,1.2,null,Hong Kong-2,null,null,", - "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + "3,1.3,1997-01-03T00:00:00.001Z,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", }; resultSetEqualTest( @@ -124,9 +124,9 @@ public void testFill() { resultSet = new String[] { - "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", - "2,1.2,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", - "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + "1,1.1,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "3,1.3,1997-01-03T00:00:00.001Z,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", }; resultSetEqualTest( @@ -134,9 +134,9 @@ public void testFill() { resultSet = new String[] { - "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", - "2,1.2,852163200001,null,null,1997-07-02,", - "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + "1,1.1,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,1997-01-02T00:00:00.001Z,null,null,1997-07-02,", + "3,1.3,1997-01-03T00:00:00.001Z,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", }; resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java index e08eb4c5f1d0a..9d56acf518fbf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.it.groupby; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -49,6 +51,7 @@ import static org.apache.iotdb.db.utils.constant.TestConstant.sum; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.count; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -75,6 +78,8 @@ public class IoTDBGroupByNaturalMonthIT { calendar.add(Calendar.MONTH, 1), i = calendar.getTimeInMillis()) { dataSet.add("insert into root.test.d1(timestamp, s1) values (" + i + ", 1)"); } + + dataSet.add("insert into root.testTimeZone.d1(timestamp, s1) values (1, 1)"); } protected static final DateFormat df = new SimpleDateFormat("MM/dd/yyyy:HH:mm:ss"); @@ -413,4 +418,27 @@ public void groupByNaturalMonthWithMixedUnit2() { null, currPrecision); } + + @Test + public void groupByNaturalMonthWithNonSystemDefaultTimeZone() { + try (ISession session = + EnvFactory.getEnv().getSessionConnection(TimeZone.getTimeZone("UTC+09:00").toZoneId())) { + + SessionDataSet sessionDataSet = + session.executeQueryStatement( + "select count(s1) from root.testTimeZone.d1 group by([2024-07-01, 2024-08-01), 1mo)"); + + int count = 0; + while (sessionDataSet.hasNext()) { + sessionDataSet.next(); + count++; + } + assertEquals(1, count); + + sessionDataSet.closeOperationHandle(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java index 111217e56857f..a15991c487683 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java @@ -110,7 +110,7 @@ public void test2() { .getConfig() .getCommonConfig() .setMaxNumberOfPointsInPage(4) - .setAvgSeriesPointNumberThreshold(2); + .setTargetChunkPointNum(2); EnvFactory.getEnv().initClusterEnvironment(); String[] expectedHeader = new String[] {TIMESTAMP_STR, count("root.sg2.d1.s1")}; String[] retArray = new String[] {"5,1,", "10,1,", "15,2,", "20,0,", "25,1,"}; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java index 6d30db0008564..5c30e95e7bf92 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java @@ -37,7 +37,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; import static org.apache.iotdb.itbase.constant.TestConstant.DATA_TYPE_STR; -import static org.apache.iotdb.itbase.constant.TestConstant.TIMESEIRES_STR; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESERIES_STR; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.VALUE_STR; import static org.junit.Assert.fail; @@ -102,7 +102,7 @@ public static void tearDown() throws Exception { @Test public void testLastQuery() { String[] expectedHeader = - new String[] {TIMESTAMP_STR, TIMESEIRES_STR, VALUE_STR, DATA_TYPE_STR}; + new String[] {TIMESTAMP_STR, TIMESERIES_STR, VALUE_STR, DATA_TYPE_STR}; String[] retArray = new String[] { "1679365910000,root.ln_1.tb_6141.11_TEXT,13,TEXT,", @@ -126,7 +126,7 @@ public void testLastQuery() { @Test public void testLastQueryOrderByTimeDesc() { String[] expectedHeader = - new String[] {TIMESTAMP_STR, TIMESEIRES_STR, VALUE_STR, DATA_TYPE_STR}; + new String[] {TIMESTAMP_STR, TIMESERIES_STR, VALUE_STR, DATA_TYPE_STR}; String[] retArray = new String[] { "1679365910000,root.ln_1.tb_6141.waterTP_DOUBLE,15.0,DOUBLE,", @@ -150,7 +150,7 @@ public void testLastQueryOrderByTimeDesc() { @Test public void testLastQuery1() { String[] expectedHeader = - new String[] {TIMESTAMP_STR, TIMESEIRES_STR, VALUE_STR, DATA_TYPE_STR}; + new String[] {TIMESTAMP_STR, TIMESERIES_STR, VALUE_STR, DATA_TYPE_STR}; String[] retArray = new String[] { "1679365910000,root.sg.`NH4-N_DOUBLE`,12.0,DOUBLE,", @@ -165,4 +165,18 @@ public void cacheHitTest() { testLastQueryOrderByTimeDesc(); testLastQuery1(); } + + @Test + public void testLastQuerySortWithLimit() { + String[] expectedHeader = + new String[] {TIMESTAMP_STR, TIMESERIES_STR, VALUE_STR, DATA_TYPE_STR}; + String[] retArray = + new String[] { + "1679477545000,root.ln_1.tb_6141.code_DOUBLE,2.0,DOUBLE,", + }; + resultSetEqualTest( + "select last * from root.ln_1.tb_6141 order by time desc, timeseries desc limit 1;", + expectedHeader, + retArray); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java index 062d5a345c3e8..995367f10ad15 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.last; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -190,4 +190,38 @@ public void testWithSLimitOrSOffset() { fail(e.getMessage()); } } + + @Test + public void testWithSortLimit() { + String[] retArray = + new String[] { + "2,root.sg.d2.s2,1.0,DOUBLE", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select last * from root.sg.** order by time desc, timeseries desc limit 1")) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(ColumnHeaderConstant.TIME) + + "," + + resultSet.getString(ColumnHeaderConstant.TIMESERIES) + + "," + + resultSet.getString(ColumnHeaderConstant.VALUE) + + "," + + resultSet.getString(ColumnHeaderConstant.DATATYPE); + assertEquals(retArray[cnt++], ans); + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastWithTTLIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastWithTTLIT.java index a09f2ade118d2..b69af5ab0172f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastWithTTLIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastWithTTLIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it.last; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java index 76e6ddce8170b..3e6f660d4f2e9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java @@ -23,16 +23,19 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.metrics.reporter.prometheus.PrometheusReporter; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.regex.Pattern; @@ -65,7 +68,13 @@ public class IoTDBMetricIT { private static final String VALID_LOG_STRING = "This line {} is invalid in prometheus line protocol"; - public static boolean isValidPrometheusTextFormat(String metrics) { + private static final String TEST_USERNAME = "good"; + private static final String TEST_PASSWORD = "??"; + + private static final String WRONG_USERNAME = "bad"; + private static final String WRONG_PASSWORD = "!!"; + + private static boolean isValidPrometheusTextFormat(String metrics) { String[] lines = metrics.split("\\n"); boolean valid = true; @@ -107,8 +116,8 @@ private static boolean isValidTypeLine(String line) { return Pattern.matches(TYPE_REGEX, line.trim()); } - @BeforeClass - public static void setUp() throws Exception { + @Before + public void setUp() throws Exception { // Start ConfigNode with Prometheus reporter up EnvFactory.getEnv() .getConfig() @@ -119,21 +128,86 @@ public static void setUp() throws Exception { .getConfig() .getDataNodeConfig() .setMetricReporterType(Collections.singletonList("PROMETHEUS")); - EnvFactory.getEnv().initClusterEnvironment(); } - @AfterClass - public static void tearDown() throws Exception { + @After + public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void testPrometheusReporterWithoutAuth() { + EnvFactory.getEnv().initClusterEnvironment(); + + List metricContents = EnvFactory.getEnv().getMetricPrometheusReporterContents(null); + for (String metricContent : metricContents) { + Assert.assertNotNull(metricContent); + Assert.assertNotEquals(0, metricContent.length()); + Assert.assertTrue(isValidPrometheusTextFormat(metricContent)); + } + } + @Test public void testPrometheusReporter() { - List metricContents = EnvFactory.getEnv().getMetricPrometheusReporterContents(); + EnvFactory.getEnv() + .getConfig() + .getConfigNodeConfig() + .setMetricPrometheusReporterUsername(base64Encode(TEST_USERNAME)) + .setMetricPrometheusReporterPassword(base64Encode(TEST_PASSWORD)); + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setMetricPrometheusReporterUsername(base64Encode(TEST_USERNAME)) + .setMetricPrometheusReporterPassword(base64Encode(TEST_PASSWORD)); + EnvFactory.getEnv().initClusterEnvironment(); + + wrongUsernameTest(); + wrongPasswordTest(); + correctUsernameAndPasswordTest(); + } + + private void wrongUsernameTest() { + List metricContents = + EnvFactory.getEnv() + .getMetricPrometheusReporterContents( + buildPrometheusReporterAuthHeader(WRONG_USERNAME, TEST_PASSWORD)); + for (String metricContent : metricContents) { + Assert.assertNull(metricContent); + } + } + + private void wrongPasswordTest() { + List metricContents = + EnvFactory.getEnv() + .getMetricPrometheusReporterContents( + buildPrometheusReporterAuthHeader(TEST_USERNAME, WRONG_PASSWORD)); + for (String metricContent : metricContents) { + Assert.assertNull(metricContent); + } + } + + private void correctUsernameAndPasswordTest() { + List metricContents = + EnvFactory.getEnv() + .getMetricPrometheusReporterContents( + buildPrometheusReporterAuthHeader(TEST_USERNAME, TEST_PASSWORD)); for (String metricContent : metricContents) { Assert.assertNotNull(metricContent); Assert.assertNotEquals(0, metricContent.length()); Assert.assertTrue(isValidPrometheusTextFormat(metricContent)); } } + + private String buildPrometheusReporterAuthHeader(String username, String password) { + if (username == null || username.isEmpty()) { + return null; + } + String raw = username + PrometheusReporter.DIVIDER_BETWEEN_USERNAME_AND_DIVIDER + password; + String base64 = Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8)); + return PrometheusReporter.BASIC_AUTH_PREFIX + base64; + } + + private static String base64Encode(String raw) { + return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8)); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java deleted file mode 100644 index c6f502bd4b54f..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java +++ /dev/null @@ -1,1406 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.it.orderBy; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.category.LocalStandaloneIT; - -import org.bouncycastle.util.Arrays; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Objects; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IoTDBOrderByForDebugIT { - - // the data can be viewed in - // https://docs.google.com/spreadsheets/d/1OWA1bKraArCwWVnuTjuhJ5yLG0PFLdD78gD6FjquepI/edit#gid=0 - private static final String[] sql = - new String[] { - "CREATE DATABASE root.sg", - "CREATE TIMESERIES root.sg.d.num WITH DATATYPE=INT32, ENCODING=RLE", - "CREATE TIMESERIES root.sg.d.bigNum WITH DATATYPE=INT64, ENCODING=RLE", - "CREATE TIMESERIES root.sg.d.floatNum WITH DATATYPE=DOUBLE, ENCODING=RLE, 'MAX_POINT_NUMBER'='5'", - "CREATE TIMESERIES root.sg.d.str WITH DATATYPE=TEXT, ENCODING=PLAIN", - "CREATE TIMESERIES root.sg.d.bool WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(0,3,2947483648,231.2121,\"coconut\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(20,2,2147483648,434.12,\"pineapple\",TRUE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(40,1,2247483648,12.123,\"apricot\",TRUE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(80,9,2147483646,43.12,\"apple\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(100,8,2147483964,4654.231,\"papaya\",TRUE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(31536000000,6,2147483650,1231.21,\"banana\",TRUE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(31536000100,10,3147483648,231.55,\"pumelo\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(31536000500,4,2147493648,213.1,\"peach\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(31536001000,5,2149783648,56.32,\"orange\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(31536010000,7,2147983648,213.112,\"lemon\",TRUE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(31536100000,11,2147468648,54.121,\"pitaya\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(41536000000,12,2146483648,45.231,\"strawberry\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(41536000020,14,2907483648,231.34,\"cherry\",FALSE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(41536900000,13,2107483648,54.12,\"lychee\",TRUE)", - "insert into root.sg.d(timestamp,num,bigNum,floatNum,str,bool) values(51536000000,15,3147483648,235.213,\"watermelon\",TRUE)", - // Newly added 'flush' command compared to IoTDBOrderByIT - "flush" - }; - - private static final String[] sql2 = - new String[] { - "CREATE TIMESERIES root.sg.d2.num WITH DATATYPE=INT32, ENCODING=RLE", - "CREATE TIMESERIES root.sg.d2.bigNum WITH DATATYPE=INT64, ENCODING=RLE", - "CREATE TIMESERIES root.sg.d2.floatNum WITH DATATYPE=DOUBLE, ENCODING=RLE, 'MAX_POINT_NUMBER'='5'", - "CREATE TIMESERIES root.sg.d2.str WITH DATATYPE=TEXT, ENCODING=PLAIN", - "CREATE TIMESERIES root.sg.d2.bool WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(0,3,2947483648,231.2121,\"coconut\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(20,2,2147483648,434.12,\"pineapple\",TRUE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(40,1,2247483648,12.123,\"apricot\",TRUE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(80,9,2147483646,43.12,\"apple\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(100,8,2147483964,4654.231,\"papaya\",TRUE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(31536000000,6,2147483650,1231.21,\"banana\",TRUE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(31536000100,10,3147483648,231.55,\"pumelo\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(31536000500,4,2147493648,213.1,\"peach\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(31536001000,5,2149783648,56.32,\"orange\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(31536010000,7,2147983648,213.112,\"lemon\",TRUE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(31536100000,11,2147468648,54.121,\"pitaya\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(41536000000,12,2146483648,45.231,\"strawberry\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(41536000020,14,2907483648,231.34,\"cherry\",FALSE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(41536900000,13,2107483648,54.12,\"lychee\",TRUE)", - "insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(51536000000,15,3147483648,235.213,\"watermelon\",TRUE)", - // Newly added 'flush' command compared to IoTDBOrderByIT - "flush" - }; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); - EnvFactory.getEnv().initClusterEnvironment(); - insertData(); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - protected static void insertData() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - for (String sql : sql) { - statement.execute(sql); - } - for (String sql : sql2) { - statement.execute(sql); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - // ordinal data - String[][] res = - new String[][] { - {"0", "3", "2947483648", "231.2121", "coconut", "false"}, - {"20", "2", "2147483648", "434.12", "pineapple", "true"}, - {"40", "1", "2247483648", "12.123", "apricot", "true"}, - {"80", "9", "2147483646", "43.12", "apple", "false"}, - {"100", "8", "2147483964", "4654.231", "papaya", "true"}, - {"31536000000", "6", "2147483650", "1231.21", "banana", "true"}, - {"31536000100", "10", "3147483648", "231.55", "pumelo", "false"}, - {"31536000500", "4", "2147493648", "213.1", "peach", "false"}, - {"31536001000", "5", "2149783648", "56.32", "orange", "false"}, - {"31536010000", "7", "2147983648", "213.112", "lemon", "true"}, - {"31536100000", "11", "2147468648", "54.121", "pitaya", "false"}, - {"41536000000", "12", "2146483648", "45.231", "strawberry", "false"}, - {"41536000020", "14", "2907483648", "231.34", "cherry", "false"}, - {"41536900000", "13", "2107483648", "54.12", "lychee", "true"}, - {"51536000000", "15", "3147483648", "235.213", "watermelon", "true"}, - }; - - private void checkHeader(ResultSetMetaData resultSetMetaData, String[] title) - throws SQLException { - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - assertEquals(title[i - 1], resultSetMetaData.getColumnName(i)); - } - } - - private void testNormalOrderBy(String sql, int[] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader( - metaData, - new String[] { - "Time", - "root.sg.d.num", - "root.sg.d.bigNum", - "root.sg.d.floatNum", - "root.sg.d.str", - "root.sg.d.bool" - }); - int i = 0; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - String actualNum = resultSet.getString(2); - String actualBigNum = resultSet.getString(3); - double actualFloatNum = resultSet.getDouble(4); - String actualStr = resultSet.getString(5); - String actualBool = resultSet.getString(6); - - assertEquals(res[ans[i]][0], actualTime); - assertEquals(res[ans[i]][1], actualNum); - assertEquals(res[ans[i]][2], actualBigNum); - assertEquals(Double.parseDouble(res[ans[i]][3]), actualFloatNum, 0.0001); - assertEquals(res[ans[i]][4], actualStr); - assertEquals(res[ans[i]][5], actualBool); - - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - // 1. One-level order by test - @Test - public void orderByTest1() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by num"; - int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest2() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by bigNum,time"; - int[] ans = {13, 11, 10, 3, 1, 5, 4, 7, 9, 8, 2, 12, 0, 6, 14}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest3() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by floatNum"; - int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest4() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by str"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest5() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by num desc"; - int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; - testNormalOrderBy(sql, Arrays.reverse(ans)); - } - - @Test - public void orderByTest6() { - String sql = - "select num,bigNum,floatNum,str,bool from root.sg.d order by bigNum desc, time asc"; - int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest7() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by floatNum desc"; - int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; - testNormalOrderBy(sql, Arrays.reverse(ans)); - } - - @Test - public void orderByTest8() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by str desc"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderBy(sql, Arrays.reverse(ans)); - } - - @Test - public void orderByTest15() { - String sql = "select num+bigNum,floatNum from root.sg.d order by str"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader( - metaData, - new String[] {"Time", "root.sg.d.num + root.sg.d.bigNum", "root.sg.d.floatNum"}); - int i = 0; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - double actualNum = resultSet.getDouble(2); - double actualFloat = resultSet.getDouble(3); - - assertEquals(res[ans[i]][0], actualTime); - assertEquals( - Long.parseLong(res[ans[i]][1]) + Long.parseLong(res[ans[i]][2]), actualNum, 0.0001); - assertEquals(Double.parseDouble(res[ans[i]][3]), actualFloat, 0.0001); - - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - // 2. Multi-level order by test - @Test - public void orderByTest9() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by bool asc, str asc"; - int[] ans = {3, 12, 0, 8, 7, 10, 6, 11, 2, 5, 9, 13, 4, 1, 14}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest10() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by bool asc, num desc"; - int[] ans = {12, 11, 10, 6, 3, 8, 7, 0, 14, 13, 4, 9, 5, 1, 2}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest11() { - String sql = - "select num,bigNum,floatNum,str,bool from root.sg.d order by bigNum desc, floatNum desc"; - int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest12() { - String sql = - "select num,bigNum,floatNum,str,bool from root.sg.d order by str desc, floatNum desc"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderBy(sql, Arrays.reverse(ans)); - } - - @Test - public void orderByTest13() { - String sql = - "select num,bigNum,floatNum,str,bool from root.sg.d order by num+floatNum desc, floatNum desc"; - int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; - testNormalOrderBy(sql, ans); - } - - @Test - public void orderByTest14() { - String sql = "select num+bigNum from root.sg.d order by num+floatNum desc, floatNum desc"; - int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader(metaData, new String[] {"Time", "root.sg.d.num + root.sg.d.bigNum"}); - int i = 0; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - double actualNum = resultSet.getDouble(2); - - assertEquals(res[ans[i]][0], actualTime); - assertEquals( - Long.parseLong(res[ans[i]][1]) + Long.parseLong(res[ans[i]][2]), actualNum, 0.001); - - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByTest16() { - String sql = "select num+floatNum from root.sg.d order by floatNum+num desc, floatNum desc"; - int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader(metaData, new String[] {"Time", "root.sg.d.num + root.sg.d.floatNum"}); - int i = 0; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - double actualNum = resultSet.getDouble(2); - - assertEquals(res[ans[i]][0], actualTime); - assertEquals( - Long.parseLong(res[ans[i]][1]) + Double.parseDouble(res[ans[i]][3]), - actualNum, - 0.001); - - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByTest17() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by str desc, str asc"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderBy(sql, Arrays.reverse(ans)); - } - - @Test - public void orderByTest18() { - String sql = "select num,bigNum,floatNum,str,bool from root.sg.d order by str, str"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderBy(sql, ans); - } - - // limit cannot be pushed down in ORDER BY - @Test - public void orderByTest19() { - String sql = "select num from root.sg.d order by num limit 5"; - int[] ans = {2, 1, 0, 7, 8}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader(metaData, new String[] {"Time", "root.sg.d.num"}); - int i = 0; - while (resultSet.next()) { - String actualTime = resultSet.getString(1); - String actualNum = resultSet.getString(2); - assertEquals(res[ans[i]][0], actualTime); - assertEquals(res[ans[i]][1], actualNum); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - // 3. aggregation query - @Test - public void orderByInAggregationTest() { - String sql = "select avg(num) from root.sg.d group by session(10000ms) order by avg(num) desc"; - double[][] ans = new double[][] {{15.0}, {13.0}, {13.0}, {11.0}, {6.4}, {4.6}}; - long[] times = - new long[] {51536000000L, 41536000000L, 41536900000L, 31536100000L, 31536000000L, 0L}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualAvg = resultSet.getDouble(2); - assertEquals(times[i], actualTime); - assertEquals(ans[i][0], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest2() { - String sql = - "select avg(num) from root.sg.d group by session(10000ms) order by max_value(floatNum)"; - double[][] ans = - new double[][] { - {13.0, 54.12}, - {11.0, 54.121}, - {13.0, 231.34}, - {15.0, 235.213}, - {6.4, 1231.21}, - {4.6, 4654.231} - }; - long[] times = - new long[] {41536900000L, 31536100000L, 41536000000L, 51536000000L, 31536000000L, 0L}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualAvg = resultSet.getDouble(2); - assertEquals(times[i], actualTime); - assertEquals(ans[i][0], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest3() { - String sql = - "select avg(num) from root.sg.d group by session(10000ms) order by avg(num) desc,max_value(floatNum)"; - double[] ans = new double[] {15.0, 13.0, 13.0, 11.0, 6.4, 4.6}; - long[] times = - new long[] {51536000000L, 41536900000L, 41536000000L, 31536100000L, 31536000000L, 0L}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualAvg = resultSet.getDouble(2); - assertEquals(times[i], actualTime); - assertEquals(ans[i], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest4() { - String sql = - "select avg(num)+avg(floatNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; - double[][] ans = - new double[][] {{1079.56122}, {395.4584}, {65.121}, {151.2855}, {67.12}, {250.213}}; - long[] times = - new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; - int[] order = new int[] {2, 4, 3, 5, 1, 0}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualAvg = resultSet.getDouble(2); - assertEquals(times[order[i]], actualTime); - assertEquals(ans[order[i]][0], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest5() { - String sql = - "select min_value(bigNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; - long[] ans = - new long[] {2147483646L, 2147483650L, 2147468648L, 2146483648L, 2107483648L, 3147483648L}; - long[] times = - new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; - int[] order = new int[] {2, 4, 3, 5, 1, 0}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - long actualMinValue = resultSet.getLong(2); - assertEquals(times[order[i]], actualTime); - assertEquals(ans[order[i]], actualMinValue, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest6() { - String sql = - "select min_value(num)+min_value(bigNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; - long[] ans = - new long[] {2147483647L, 2147483654L, 2147468659L, 2146483660L, 2107483661L, 3147483663L}; - long[] times = - new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; - int[] order = new int[] {2, 4, 3, 5, 1, 0}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualMinValue = resultSet.getDouble(2); - assertEquals(times[order[i]], actualTime); - assertEquals(ans[order[i]], actualMinValue, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest7() { - String sql = - "select avg(num)+min_value(floatNum) from root.sg.d group by session(10000ms) order by max_value(floatNum)"; - double[][] ans = - new double[][] { - {13.0, 54.12, 54.12}, - {11.0, 54.121, 54.121}, - {13.0, 231.34, 45.231}, - {15.0, 235.213, 235.213}, - {6.4, 1231.21, 56.32}, - {4.6, 4654.231, 12.123} - }; - long[] times = - new long[] {41536900000L, 31536100000L, 41536000000L, 51536000000L, 31536000000L, 0L}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualAvg = resultSet.getDouble(2); - assertEquals(times[i], actualTime); - assertEquals(ans[i][0] + ans[i][2], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationTest8() { - String sql = - "select avg(num)+avg(floatNum) from root.sg.d group by session(10000ms) order by avg(floatNum)+avg(num)"; - double[][] ans = - new double[][] {{1079.56122}, {395.4584}, {65.121}, {151.2855}, {67.12}, {250.213}}; - long[] times = - new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; - int[] order = new int[] {2, 4, 3, 5, 1, 0}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - double actualAvg = resultSet.getDouble(2); - assertEquals(times[order[i]], actualTime); - assertEquals(ans[order[i]][0], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - // 4. raw data query with align by device - private void testNormalOrderByAlignByDevice(String sql, int[] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader( - metaData, new String[] {"Time", "Device", "num", "bigNum", "floatNum", "str", "bool"}); - int i = 0; - int total = 0; - String device = "root.sg.d"; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - String actualDevice = resultSet.getString(2); - String actualNum = resultSet.getString(3); - String actualBigNum = resultSet.getString(4); - double actualFloatNum = resultSet.getDouble(5); - String actualStr = resultSet.getString(6); - String actualBool = resultSet.getString(7); - - assertEquals(device, actualDevice); - assertEquals(res[ans[i]][0], actualTime); - assertEquals(res[ans[i]][1], actualNum); - assertEquals(res[ans[i]][2], actualBigNum); - assertEquals(Double.parseDouble(res[ans[i]][3]), actualFloatNum, 0.0001); - assertEquals(res[ans[i]][4], actualStr); - assertEquals(res[ans[i]][5], actualBool); - - if (device.equals("root.sg.d")) { - device = "root.sg.d2"; - } else { - device = "root.sg.d"; - i++; - } - total++; - } - assertEquals(i, ans.length); - assertEquals(total, ans.length * 2); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void alignByDeviceOrderByTest1() { - String sql = - "select num+bigNum from root.** order by num+floatNum desc, floatNum desc align by device"; - int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; - String device = "root.sg.d"; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - String actualTime = resultSet.getString(1); - String actualDevice = resultSet.getString(2); - double actualNum = resultSet.getDouble(3); - - assertEquals(device, actualDevice); - assertEquals(res[ans[i]][0], actualTime); - assertEquals( - Long.parseLong(res[ans[i]][1]) + Long.parseLong(res[ans[i]][2]), actualNum, 0.0001); - if (device.equals("root.sg.d")) { - device = "root.sg.d2"; - } else { - device = "root.sg.d"; - i++; - } - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void alignByDeviceOrderByTest2() { - String sql = "select num,bigNum,floatNum,str,bool from root.** order by num align by device"; - int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; - testNormalOrderByAlignByDevice(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest3() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by floatNum align by device"; - int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; - testNormalOrderByAlignByDevice(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest4() { - String sql = "select num,bigNum,floatNum,str,bool from root.** order by str align by device"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderByAlignByDevice(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest5() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by num desc align by device"; - int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; - testNormalOrderByAlignByDevice(sql, Arrays.reverse(ans)); - } - - @Test - public void alignByDeviceOrderByTest6() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by str desc align by device"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderByAlignByDevice(sql, Arrays.reverse(ans)); - } - - @Test - public void alignByDeviceOrderByTest7() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by bool asc, num desc align by device"; - int[] ans = {12, 11, 10, 6, 3, 8, 7, 0, 14, 13, 4, 9, 5, 1, 2}; - testNormalOrderByAlignByDevice(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest8() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by bigNum desc, floatNum desc align by device"; - int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - testNormalOrderByAlignByDevice(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest9() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by str desc, floatNum desc align by device"; - int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; - testNormalOrderByAlignByDevice(sql, Arrays.reverse(ans)); - } - - private void testNormalOrderByMixAlignBy(String sql, int[] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader( - metaData, new String[] {"Time", "Device", "num", "bigNum", "floatNum", "str", "bool"}); - int i = 0; - int total = 0; - String device = "root.sg.d"; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - String actualDevice = resultSet.getString(2); - String actualNum = resultSet.getString(3); - String actualBigNum = resultSet.getString(4); - double actualFloatNum = resultSet.getDouble(5); - String actualStr = resultSet.getString(6); - String actualBool = resultSet.getString(7); - - assertEquals(device, actualDevice); - assertEquals(res[ans[i]][0], actualTime); - assertEquals(res[ans[i]][1], actualNum); - assertEquals(res[ans[i]][2], actualBigNum); - assertEquals(Double.parseDouble(res[ans[i]][3]), actualFloatNum, 0.0001); - assertEquals(res[ans[i]][4], actualStr); - assertEquals(res[ans[i]][5], actualBool); - - if (device.equals("root.sg.d2")) { - i++; - device = "root.sg.d"; - } else { - device = "root.sg.d2"; - } - - total++; - } - assertEquals(total, ans.length * 2); - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - private void testDeviceViewOrderByMixAlignBy(String sql, int[] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader( - metaData, new String[] {"Time", "Device", "num", "bigNum", "floatNum", "str", "bool"}); - int i = 0; - int total = 0; - String device = "root.sg.d2"; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - String actualDevice = resultSet.getString(2); - String actualNum = resultSet.getString(3); - String actualBigNum = resultSet.getString(4); - double actualFloatNum = resultSet.getDouble(5); - String actualStr = resultSet.getString(6); - String actualBool = resultSet.getString(7); - - assertEquals(device, actualDevice); - assertEquals(res[ans[i]][0], actualTime); - assertEquals(res[ans[i]][1], actualNum); - assertEquals(res[ans[i]][2], actualBigNum); - assertEquals(Double.parseDouble(res[ans[i]][3]), actualFloatNum, 0.0001); - assertEquals(res[ans[i]][4], actualStr); - assertEquals(res[ans[i]][5], actualBool); - - i++; - total++; - if (total == ans.length) { - i = 0; - if (device.equals("root.sg.d2")) { - device = "root.sg.d"; - } else { - device = "root.sg.d2"; - } - } - } - assertEquals(total, ans.length * 2); - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - private void orderByBigNumAlignByDevice(String sql, int[] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - checkHeader( - metaData, new String[] {"Time", "Device", "num", "bigNum", "floatNum", "str", "bool"}); - int i = 0; - int total = 0; - String device = "root.sg.d"; - while (resultSet.next()) { - - String actualTime = resultSet.getString(1); - String actualDevice = resultSet.getString(2); - String actualNum = resultSet.getString(3); - String actualBigNum = resultSet.getString(4); - double actualFloatNum = resultSet.getDouble(5); - String actualStr = resultSet.getString(6); - String actualBool = resultSet.getString(7); - - if (total < 4) { - i = total % 2; - if (total < 2) { - device = "root.sg.d2"; - } else { - device = "root.sg.d"; - } - } - - assertEquals(device, actualDevice); - assertEquals(res[ans[i]][0], actualTime); - assertEquals(res[ans[i]][1], actualNum); - assertEquals(res[ans[i]][2], actualBigNum); - assertEquals(Double.parseDouble(res[ans[i]][3]), actualFloatNum, 0.0001); - assertEquals(res[ans[i]][4], actualStr); - assertEquals(res[ans[i]][5], actualBool); - - if (device.equals("root.sg.d2")) { - device = "root.sg.d"; - } else { - i++; - device = "root.sg.d2"; - } - - total++; - } - assertEquals(i, ans.length); - assertEquals(total, ans.length * 2); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void alignByDeviceOrderByTest12() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by bigNum desc, device desc, time asc align by device"; - int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - orderByBigNumAlignByDevice(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest13() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by bigNum desc, time desc, device asc align by device"; - int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - testNormalOrderByMixAlignBy(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest14() { - int[] ans = {14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by time desc, bigNum desc, device asc align by device"; - testNormalOrderByMixAlignBy(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest15() { - int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by device desc, bigNum desc, time asc align by device"; - testDeviceViewOrderByMixAlignBy(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest16() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by device desc, time asc, bigNum desc align by device"; - int[] ans = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; - testDeviceViewOrderByMixAlignBy(sql, ans); - } - - @Test - public void alignByDeviceOrderByTest17() { - String sql = - "select num,bigNum,floatNum,str,bool from root.** order by bigNum desc, device desc, num asc, time asc align by device"; - int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; - orderByBigNumAlignByDevice(sql, ans); - } - - // 5. aggregation query align by device - @Test - public void orderByInAggregationAlignByDeviceTest() { - String sql = - "select avg(num) from root.** group by session(10000ms) order by avg(num) align by device"; - - double[] ans = {4.6, 4.6, 6.4, 6.4, 11.0, 11.0, 13.0, 13.0, 13.0, 13.0, 15.0, 15.0}; - long[] times = - new long[] { - 0L, - 0L, - 31536000000L, - 31536000000L, - 31536100000L, - 31536100000L, - 41536000000L, - 41536900000L, - 41536000000L, - 41536900000L, - 51536000000L, - 51536000000L - }; - String[] device = - new String[] { - "root.sg.d", - "root.sg.d2", - "root.sg.d", - "root.sg.d2", - "root.sg.d", - "root.sg.d2", - "root.sg.d", - "root.sg.d", - "root.sg.d2", - "root.sg.d2", - "root.sg.d", - "root.sg.d2" - }; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - long actualTime = resultSet.getLong(1); - String actualDevice = resultSet.getString(2); - double actualAvg = resultSet.getDouble(3); - - assertEquals(device[i], actualDevice); - assertEquals(times[i], actualTime); - assertEquals(ans[i], actualAvg, 0.0001); - i++; - } - assertEquals(i, ans.length); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationAlignByDeviceTest2() { - String sql = "select avg(num) from root.** order by avg(num) align by device"; - String value = "8"; - checkSingleDouble(sql, value, true); - } - - private void checkSingleDouble(String sql, Object value, boolean deviceAsc) { - String device = "root.sg.d"; - if (!deviceAsc) device = "root.sg.d2"; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - String deviceName = resultSet.getString(1); - double actualVal = resultSet.getDouble(2); - assertEquals(deviceName, device); - assertEquals(Double.parseDouble(value.toString()), actualVal, 1); - if (device.equals("root.sg.d")) device = "root.sg.d2"; - else device = "root.sg.d"; - i++; - } - assertEquals(i, 2); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByInAggregationAlignByDeviceTest3() { - String sql = - "select avg(num)+avg(bigNum) from root.** order by max_value(floatNum) align by device"; - long value = 2388936669L + 8; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest4() { - - String sql = - "select avg(num)+avg(bigNum) from root.** order by max_value(floatNum)+min_value(num) align by device"; - long value = 2388936669L + 8; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest5() { - String sql = - "select avg(num) from root.** order by max_value(floatNum)+avg(num) align by device"; - String value = "8"; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest6() { - String sql = - "select avg(num) from root.** order by max_value(floatNum)+avg(num), device asc, time desc align by device"; - String value = "8"; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest7() { - String sql = - "select avg(num) from root.** order by max_value(floatNum)+avg(num), time asc, device desc align by device"; - String value = "8"; - checkSingleDouble(sql, value, false); - } - - @Test - public void orderByInAggregationAlignByDeviceTest8() { - String sql = - "select avg(num) from root.** order by time asc, max_value(floatNum)+avg(num), device desc align by device"; - String value = "8"; - checkSingleDouble(sql, value, false); - } - - @Test - public void orderByInAggregationAlignByDeviceTest9() { - String sql = - "select avg(num) from root.** order by device asc, max_value(floatNum)+avg(num), time desc align by device"; - String value = "8"; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest10() { - String sql = - "select avg(num) from root.** order by max_value(floatNum) desc,time asc, avg(num) asc, device desc align by device"; - String value = "8"; - checkSingleDouble(sql, value, false); - } - - @Test - public void orderByInAggregationAlignByDeviceTest11() { - String sql = - "select avg(num) from root.** order by max_value(floatNum) desc,device asc, avg(num) asc, time desc align by device"; - String value = "8"; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest12() { - String sql = - "select avg(num+floatNum) from root.** order by time,avg(num+floatNum) align by device"; - String value = "537.34154"; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest13() { - String sql = "select avg(num) from root.** order by time,avg(num+floatNum) align by device"; - String value = "8"; - checkSingleDouble(sql, value, true); - } - - @Test - public void orderByInAggregationAlignByDeviceTest14() { - String sql = "select avg(num+floatNum) from root.** order by time,avg(num) align by device"; - String value = "537.34154"; - checkSingleDouble(sql, value, true); - } - - String[][] UDFRes = - new String[][] { - {"0", "3", "0", "0"}, - {"20", "2", "0", "0"}, - {"40", "1", "0", "0"}, - {"80", "9", "0", "0"}, - {"100", "8", "0", "0"}, - {"31536000000", "6", "0", "0"}, - {"31536000100", "10", "0", "0"}, - {"31536000500", "4", "0", "0"}, - {"31536001000", "5", "0", "0"}, - {"31536010000", "7", "0", "0"}, - {"31536100000", "11", "0", "0"}, - {"41536000000", "12", "2146483648", "0"}, - {"41536000020", "14", "0", "14"}, - {"41536900000", "13", "2107483648", "0"}, - {"51536000000", "15", "0", "15"}, - }; - - // UDF Test - private void orderByUDFTest(String sql, int[] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - String time = resultSet.getString(1); - String num = resultSet.getString(2); - String topK = resultSet.getString(3); - String bottomK = resultSet.getString(4); - - assertEquals(time, UDFRes[ans[i]][0]); - assertEquals(num, UDFRes[ans[i]][1]); - if (Objects.equals(UDFRes[ans[i]][3], "0")) { - assertNull(topK); - } else { - assertEquals(topK, UDFRes[ans[i]][3]); - } - - if (Objects.equals(UDFRes[ans[i]][2], "0")) { - assertNull(bottomK); - } else { - assertEquals(bottomK, UDFRes[ans[i]][2]); - } - - i++; - } - assertEquals(i, 15); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void orderByUDFTest1() { - String sql = - "select num, top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2') from root.sg.d order by top_k(num, 'k'='2') nulls first, bottom_k(bigNum, 'k'='2') nulls first"; - int[] ans = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 11, 12, 14}; - orderByUDFTest(sql, ans); - } - - @Test - public void orderByUDFTest2() { - String sql = - "select num, top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2') from root.sg.d order by top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2')"; - int[] ans = {12, 14, 13, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - orderByUDFTest(sql, ans); - } - - private void errorTest(String sql, String error) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.executeQuery(sql); - } catch (Exception e) { - assertEquals(error, e.getMessage()); - } - } - - @Test - public void errorTest1() { - errorTest( - "select num from root.sg.d order by avg(bigNum)", - "701: Raw data and aggregation hybrid query is not supported."); - } - - @Test - public void errorTest2() { - errorTest( - "select avg(num) from root.sg.d order by bigNum", - "701: Raw data and aggregation hybrid query is not supported."); - } - - @Test - public void errorTest3() { - errorTest( - "select bigNum,floatNum from root.sg.d order by s1", - "701: root.sg.d.s1 in order by clause doesn't exist."); - } - - @Test - public void errorTest4() { - errorTest( - "select bigNum,floatNum from root.** order by bigNum", - "701: root.**.bigNum in order by clause shouldn't refer to more than one timeseries."); - } - - @Test - public void errorTest5() { - errorTest( - "select bigNum,floatNum from root.** order by s1 align by device", - "701: s1 in order by clause doesn't exist."); - } - - @Test - public void errorTest6() { - errorTest( - "select bigNum,floatNum from root.** order by root.sg.d.bigNum align by device", - "701: ALIGN BY DEVICE: the suffix paths can only be measurement or one-level wildcard"); - } - - @Test - public void errorTest7() { - errorTest( - "select last bigNum,floatNum from root.** order by root.sg.d.bigNum", - "701: root.sg.d.bigNum in order by clause doesn't exist in the result of last query."); - } - - // last query - public void testLastQueryOrderBy(String sql, String[][] ans) { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - int i = 0; - while (resultSet.next()) { - String time = resultSet.getString(1); - String num = resultSet.getString(2); - String value = resultSet.getString(3); - String dataType = resultSet.getString(4); - - assertEquals(time, ans[0][i]); - assertEquals(num, ans[1][i]); - assertEquals(value, ans[2][i]); - assertEquals(dataType, ans[3][i]); - - i++; - } - assertEquals(i, 4); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void lastQueryOrderBy() { - String[][] ans = - new String[][] { - {"51536000000", "51536000000", "51536000000", "51536000000"}, - {"root.sg.d.num", "root.sg.d2.num", "root.sg.d.bigNum", "root.sg.d2.bigNum"}, - {"15", "15", "3147483648", "3147483648"}, - {"INT32", "INT32", "INT64", "INT64"} - }; - String sql = "select last bigNum,num from root.** order by value, timeseries"; - testLastQueryOrderBy(sql, ans); - } - - @Test - public void lastQueryOrderBy2() { - String[][] ans = - new String[][] { - {"51536000000", "51536000000", "51536000000", "51536000000"}, - {"root.sg.d2.num", "root.sg.d2.bigNum", "root.sg.d.num", "root.sg.d.bigNum"}, - {"15", "3147483648", "15", "3147483648"}, - {"INT32", "INT64", "INT32", "INT64"} - }; - String sql = "select last bigNum,num from root.** order by timeseries desc"; - testLastQueryOrderBy(sql, ans); - } - - @Test - public void lastQueryOrderBy3() { - String[][] ans = - new String[][] { - {"51536000000", "51536000000", "51536000000", "51536000000"}, - {"root.sg.d2.num", "root.sg.d2.bigNum", "root.sg.d.num", "root.sg.d.bigNum"}, - {"15", "3147483648", "15", "3147483648"}, - {"INT32", "INT64", "INT32", "INT64"} - }; - String sql = "select last bigNum,num from root.** order by timeseries desc, value asc"; - testLastQueryOrderBy(sql, ans); - } - - @Test - public void lastQueryOrderBy4() { - String[][] ans = - new String[][] { - {"51536000000", "51536000000", "51536000000", "51536000000"}, - {"root.sg.d2.num", "root.sg.d.num", "root.sg.d2.bigNum", "root.sg.d.bigNum"}, - {"15", "15", "3147483648", "3147483648"}, - {"INT32", "INT32", "INT64", "INT64"} - }; - String sql = "select last bigNum,num from root.** order by value, timeseries desc"; - testLastQueryOrderBy(sql, ans); - } - - @Test - public void lastQueryOrderBy5() { - String[][] ans = - new String[][] { - {"51536000000", "51536000000", "51536000000", "51536000000"}, - {"root.sg.d2.num", "root.sg.d.num", "root.sg.d2.bigNum", "root.sg.d.bigNum"}, - {"15", "15", "3147483648", "3147483648"}, - {"INT32", "INT32", "INT64", "INT64"} - }; - String sql = "select last bigNum,num from root.** order by datatype, timeseries desc"; - testLastQueryOrderBy(sql, ans); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java index 0c788782f2ab6..4cba7008a229b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.path; import org.apache.iotdb.it.env.EnvFactory; @@ -101,15 +102,16 @@ public void test() { @Test public void testIllegalStorageGroup() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE root.`\"ln`"); - } catch (SQLException e) { + } catch (final SQLException e) { Assert.assertTrue( e.getMessage().contains("Error StorageGroup name") || e.getMessage() - .contains("The database name can only be characters, numbers and underscores.")); - } catch (Exception e) { + .contains( + "The database name can only contain english or chinese characters, numbers, backticks and underscores.")); + } catch (final Exception e) { e.printStackTrace(); Assert.fail(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java index 37f5b38bcdc85..e2c60bd644e0b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java @@ -135,6 +135,20 @@ public void testKind2Basic() { resultSetEqualTest(sql, expectedHeader, retArray); } + @Test + public void testShortCircuitEvaluation() { + String[] retArray = + new String[] {"0,0.0,", "1000000,11.0,", "20000000,22.0,", "210000000,33.0,"}; + String[] expectedHeader = + new String[] { + "Time", "CASE WHEN 1 = 0 THEN root.sg.d1.s1 / 0 WHEN 1 != 0 THEN root.sg.d1.s1 END", + }; + resultSetEqualTest( + "select case when 1=0 then s1/0 when 1!=0 then s1 end from root.sg.d1", + expectedHeader, + retArray); + } + @Test public void testKind1InputTypeRestrict() { // WHEN clause must return BOOLEAN @@ -873,4 +887,36 @@ public void testKind2CaseInCase() { }; resultSetEqualTest(sql, expectedHeader, retArray); } + + @Test + public void testKind1Logic() { + String sql = + "select case when s3 >= 0 and s3 < 20 and s4 >= 50 and s4 < 60 then 'just so so~~~' when s3 >= 20 and s3 < 40 and s4 >= 70 and s4 < 80 then 'very well~~~' end as result from root.sg.d2"; + String[] expectedHeader = new String[] {TIMESTAMP_STR, "result"}; + String[] retArray = + new String[] { + "0,null,", "1000000,just so so~~~,", "20000000,null,", "210000000,very well~~~,", + }; + resultSetEqualTest(sql, expectedHeader, retArray); + } + + @Test + public void testMultipleSatisfyCase() { + // Test the result when two when clause are satisfied + String sql = + "select case when s3 < 20 or s4 > 60 then \"just so so~~~\" when s3 > 20 or s4 < 60 then \"very well~~~\" end from root.sg.d2"; + String[] expectedHeader = + new String[] { + TIMESTAMP_STR, + "CASE WHEN root.sg.d2.s3 < 20 | root.sg.d2.s4 > 60 THEN \"just so so~~~\" WHEN root.sg.d2.s3 > 20 | root.sg.d2.s4 < 60 THEN \"very well~~~\" END" + }; + String[] retArray = + new String[] { + "0,just so so~~~,", + "1000000,just so so~~~,", + "20000000,just so so~~~,", + "210000000,just so so~~~,", + }; + resultSetEqualTest(sql, expectedHeader, retArray); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java new file mode 100644 index 0000000000000..c8225eeced5f6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java @@ -0,0 +1,674 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@Ignore +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBEncryptionValueQueryIT { + private static String[] sqls = + new String[] { + "CREATE DATABASE root.ln", + "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465600000,true)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465660000,true)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465720000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465780000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465840000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465900000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465960000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466020000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466080000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466140000,false)", + "create timeseries root.ln.wf01.wt01.temperature with datatype=FLOAT,encoding=RLE", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600000,25.957603)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465660000,24.359503)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465720000,20.092794)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465780000,20.182663)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465840000,21.125198)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465900000,22.720892)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465960000,20.71);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466020000,21.451046);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466080000,22.57987);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466140000,20.98177);", + "create timeseries root.ln.wf02.wt02.hardware with datatype=TEXT,encoding=PLAIN", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465600000,'v2')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465660000,'v2')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465720000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465780000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465840000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465900000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465960000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466020000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466080000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466140000,'v1')", + "create timeseries root.ln.wf02.wt02.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465600000,true)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465660000,true)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465720000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465780000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465840000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465900000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465960000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466020000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466080000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466140000,false)", + "flush", + "CREATE DATABASE root.sgcc", + "create timeseries root.sgcc.wf03.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465600000,true)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465660000,true)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465720000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465780000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465840000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465900000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465960000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466020000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466080000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466140000,false)", + "create timeseries root.sgcc.wf03.wt01.temperature with datatype=FLOAT,encoding=RLE", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465600000,25.957603)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465660000,24.359503)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465720000,20.092794)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465780000,20.182663)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465840000,21.125198)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465900000,22.720892)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465960000,20.71)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466020000,21.451046)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466080000,22.57987)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466140000,20.98177)", + "flush", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEncryptType("UNENCRYPTED"); + EnvFactory.getEnv().initClusterEnvironment(); + importData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void importData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectTest() { + String[] retArray = + new String[] { + "1509465600000,true,25.96,v2,true,true,25.96,", + "1509465660000,true,24.36,v2,true,true,24.36,", + "1509465720000,false,20.09,v1,false,false,20.09,", + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + "1509466080000,false,22.58,v1,false,false,22.58,", + "1509466140000,false,20.98,v1,false,false,20.98,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = statement.executeQuery("select * from root.** where time>10"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void LimitTest() { + String[] retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // test 1: fetchSize < limitNumber + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + + ResultSet resultSet = + statement.executeQuery("select * from root.** where time>10 limit 5 offset 3"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + // test 1: fetchSize > limitNumber + statement.setFetchSize(10000); + Assert.assertEquals(10000, statement.getFetchSize()); + resultSet = statement.executeQuery("select * from root.** where time>10 limit 5 offset 3"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void InTest() { + String[] retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // test 1: fetchSize < limitNumber + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + ResultSet resultSet = + statement.executeQuery( + "select * from root.** where time in (1509465780000, 1509465840000, 1509465900000, 1509465960000, 1509466020000)"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + retArray = + new String[] { + "1509465600000,true,25.96,v2,true,true,25.96,", + "1509465660000,true,24.36,v2,true,true,24.36,", + "1509465720000,false,20.09,v1,false,false,20.09,", + "1509466080000,false,22.58,v1,false,false,22.58,", + "1509466140000,false,20.98,v1,false,false,20.98,", + }; + resultSet = + statement.executeQuery( + "select * from root.** where time not in (1509465780000, 1509465840000, 1509465900000, 1509465960000, 1509466020000)"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466080000,false,22.58,v1,false,false,22.58,", + }; + + resultSet = + statement.executeQuery( + "select * from root.** where root.ln.wf01.wt01.temperature in (20.18, 20.71, 22.58)"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } + + @Test + public void testRightTextQuery() { + // Text type uses the equal operator to query the correct result + String[] retArray = + new String[] { + "1509465600000,v2,", "1509465660000,v2,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + ResultSet resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf02.wt02.hardware,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + resultSetMetaData = resultSet.getMetaData(); + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + } catch (Exception e) { + Assert.assertNull(e.getMessage()); + } + } + + @Test + public void RegexpTest() { + String[] retArray = + new String[] { + "1509465600000,v2,true,", "1509465660000,v2,true,", "1509465720000,v1,false,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + // Matches a string consisting of one lowercase letter and one digit. such as: "v1","v2" + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp '^[a-z][0-9]$' and time < 1509465780000"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + + retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp 'v*' "); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void RegexpNonExistTest() { + + // Match nonexistent string.'s.' is indicates that the starting with s and the last is any + // single character + String[] retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp 's.' "); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(0, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBFuzzyQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBFuzzyQueryIT.java index a9e6e37dd565d..2694faaf52660 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBFuzzyQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBFuzzyQueryIT.java @@ -86,7 +86,7 @@ private static void initCreateSQLStatement() { sqls.add( "insert into root.t1.wf01.wt01 (time,status,temperature) values (1509466080000,'123%',18.3)"); sqls.add( - "insert into root.t1.wf01.wt01 (time,status,temperature) values (1509466090000,'\\\\',10.3)"); + "insert into root.t1.wf01.wt01 (time,status,temperature) values (1509466090000,'\\\\\\\\',10.3)"); sqls.add("insert into root.t1.wf01.wt02 (time,status) values (1509465600000,'14')"); } @@ -106,13 +106,15 @@ public void testLike() throws SQLException { Connection connection = EnvFactory.getEnv().getConnection(); Statement st0 = connection.createStatement(); ResultSet resultSet = - st0.executeQuery("select status from root.t1.wf01.wt01 where status like '1'"); + st0.executeQuery("select status from root.t1.wf01.wt01 where status not like '1'"); - Assert.assertEquals("1", outputResultStr(resultSet)); + Assert.assertEquals( + "14,616,626,6116,6%16,8[sS]*,%123,123%,\\\\\\\\", outputResultStr(resultSet)); resultSet = st0.executeQuery("select status from root.t1.wf01.wt01 where status like '%'"); - Assert.assertEquals("1,14,616,626,6116,6%16,8[sS]*,%123,123%,\\\\", outputResultStr(resultSet)); + Assert.assertEquals( + "1,14,616,626,6116,6%16,8[sS]*,%123,123%,\\\\\\\\", outputResultStr(resultSet)); resultSet = st0.executeQuery("select status from root.t1.wf01.wt01 where status like '1%'"); @@ -134,22 +136,28 @@ public void testLike() throws SQLException { Assert.assertEquals("6116,6%16", outputResultStr(resultSet)); - resultSet = st0.executeQuery("select status from root.t1.wf01.wt01 where status like '6\\%%'"); + resultSet = + st0.executeQuery( + "select status from root.t1.wf01.wt01 where status like '6\\%%' escape '\\'"); Assert.assertEquals("6%16", outputResultStr(resultSet)); - resultSet = st0.executeQuery("select status from root.t1.wf01.wt01 where status like '\\%%'"); + resultSet = + st0.executeQuery( + "select status from root.t1.wf01.wt01 where status like '\\%%' escape '\\'"); Assert.assertEquals("%123", outputResultStr(resultSet)); - resultSet = st0.executeQuery("select status from root.t1.wf01.wt01 where status like '%\\%'"); + resultSet = + st0.executeQuery( + "select status from root.t1.wf01.wt01 where status like '%\\%' escape '\\'"); Assert.assertEquals("123%", outputResultStr(resultSet)); resultSet = st0.executeQuery("select status from root.t1.wf01.wt01 where status like '%\\\\\\\\%'"); - Assert.assertEquals("\\\\", outputResultStr(resultSet)); + Assert.assertEquals("\\\\\\\\", outputResultStr(resultSet)); } @Test(expected = Exception.class) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java new file mode 100644 index 0000000000000..a9aa077466aed --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.it.env.EnvFactory; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IoTDBLoadEncryptedTsFileIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEncryptType("UNENCRYPTED"); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadSameWayEncryptedTsFileTest() { + String[] retArray = + new String[] { + "2,1,", "3,1,", "4,1,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg"); + statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateSameWayEncryptedFile(); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg.d1"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.testsg.d1.s1,", + new int[] { + Types.TIMESTAMP, Types.INTEGER, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage() + .contains("TSFile encryption is enabled, and the Load TSFile function is disabled")); + } + } + + private File generateSameWayEncryptedFile() throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + config.setEncryptType("UNENCRYPTED"); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile, config)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema("s1", TSDataType.INT32), + EncryptUtils.getEncryptParameter(config)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + config.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED"); + return tsfile; + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullOperandIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullOperandIT.java index 375289953cd2f..f292b162155f5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullOperandIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullOperandIT.java @@ -104,7 +104,7 @@ public void testCompareOperations() { "root.test.sg1.s1 = root.test.sg1.s2", "root.test.sg1.s1 > root.test.sg1.s2", "root.test.sg1.s1 < root.test.sg1.s2", - "root.test.sg1.s5 LIKE '^test$'", + "root.test.sg1.s5 LIKE pattern = 'test'", "root.test.sg1.s2 IN (1,2)", "root.test.sg1.s2 BETWEEN 1 AND 3", }; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java index cb074e324a9a0..2397e49c1dc04 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java @@ -18,11 +18,13 @@ */ package org.apache.iotdb.db.it.query; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -41,6 +43,7 @@ import java.util.List; import java.util.Map; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -465,18 +468,6 @@ private List checkHeader( return actualIndexToExpectedIndexList; } - @Test - public void testWrongTextQuery() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.executeQuery("select * from root.ln.wf02.wt02 where hardware > 'v1'"); - } catch (Exception e) { - Assert.assertEquals( - e.getMessage(), - "411: Error occurred in query process: For Basic operator,TEXT type only support EQUAL or NOTEQUAL operator"); - } - } - @Test public void testRightTextQuery() { // Text type uses the equal operator to query the correct result @@ -513,6 +504,25 @@ public void testRightTextQuery() { } Assert.assertEquals(2, cnt); + resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + resultSetMetaData = resultSet.getMetaData(); + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + } catch (Exception e) { Assert.assertNull(e.getMessage()); } @@ -661,4 +671,26 @@ public void RegexpNonExistTest() { fail(e.getMessage()); } } + + @Test + public void selectWithTimeTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeRawDataQuery( + ImmutableList.of("root.ln.wf01.wt01.time", "root.ln.wf01.wt01.temperature"), 0, 100); + + fail(); + } catch (Exception e) { + e.getMessage().contains("509: root.ln.wf01.wt01.time is not a legal path"); + } + + String expectedErrMsg = + "701: Time column is no need to appear in SELECT Clause explicitly, it will always be returned if possible"; + assertTestFail("select time from root.ln.wf01.wt01", expectedErrMsg); + assertTestFail("select time, temperature from root.ln.wf01.wt01", expectedErrMsg); + assertTestFail("select time from root.ln.wf01.wt01 where temperature > 1", expectedErrMsg); + // parse error when process 'wt01.time' + assertTestFail( + "select wt01.time, wt01.temperature from root.ln.wf01", + "700: Error occurred while parsing SQL to physical plan"); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBResultSetIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBResultSetIT.java index 1385d42872d62..33648973284a5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBResultSetIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBResultSetIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.query; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAutoCreateSchemaIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAutoCreateSchemaIT.java index 8d0380bba72b7..5453fdc9ab1b5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAutoCreateSchemaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAutoCreateSchemaIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java index 6a00ece6783e7..7fdeff084c36f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateDatabaseIT.java new file mode 100644 index 0000000000000..63e0efd2f3cb9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateDatabaseIT.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.schema; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.util.AbstractSchemaIT; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.fail; + +/** + * Notice that, all test begins with "IoTDB" is integration test. All test which will start the + * IoTDB server should be defined as integration test. + */ +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBCreateDatabaseIT extends AbstractSchemaIT { + + public IoTDBCreateDatabaseIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + + @Parameterized.BeforeParam + public static void before() throws Exception { + setUpEnvironment(); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Parameterized.AfterParam + public static void after() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + tearDownEnvironment(); + } + + @After + public void tearDown() throws Exception { + clearSchema(); + } + + /** The test creates three databases */ + @Test + public void testCreateDatabase() throws Exception { + final String[] databases = {"root.sg1", "root.sg2", "root.sg3"}; + try (final Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (final String database : databases) { + statement.execute(String.format("create database %s", database)); + } + + // ensure that current Database in cache is right. + createDatabaseTool(statement, databases); + } + // todo test restart + // EnvironmentUtils.stopDaemon(); + // setUp(); + // + // // ensure Database in cache is right after recovering. + // createDatabaseTool(Databases); + } + + private void createDatabaseTool(final Statement statement, final String[] Databases) + throws SQLException { + + List resultList = new ArrayList<>(); + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + while (resultSet.next()) { + final String databasePath = resultSet.getString(ColumnHeaderConstant.DATABASE); + resultList.add(databasePath); + } + } + Assert.assertEquals(3, resultList.size()); + + resultList = resultList.stream().sorted().collect(Collectors.toList()); + + Assert.assertEquals(Databases[0], resultList.get(0)); + Assert.assertEquals(Databases[1], resultList.get(1)); + Assert.assertEquals(Databases[2], resultList.get(2)); + } + + /** Test creating a database that path is an existence database */ + @Test + public void testCreateExistDatabase1() throws Exception { + final String database = "root.db"; + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("CREATE DATABASE %s", database)); + + try { + statement.execute(String.format("create database %s", database)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() + + ": root.db has already been created as database", + e.getMessage()); + } + } + } + + /** Test the parent node has been set as a database */ + @Test + public void testCreateExistDatabase2() throws Exception { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database root.db"); + + try { + statement.execute("create database root.db.`device`"); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() + + ": root.db has already been created as database", + e.getMessage()); + } + } + } + + /** Test creating a database exceeding 64 letters */ + @Test + public void testCreateTooLongDatabase() throws Exception { + final String database = + "root.thisDatabaseNameHasExceeded64BySevenAndCanNotBeSuccessfullyCreated"; + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + try { + statement.execute(String.format("create database %s", database)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + TSStatusCode.ILLEGAL_PATH.getStatusCode() + + ": root.thisDatabaseNameHasExceeded64BySevenAndCanNotBeSuccessfullyCreated is not a legal path, because the length of database name shall not exceed 64", + e.getMessage()); + } + } + } + + /** Test creating a "root" database */ + @Test + public void testCreateRootDatabase() throws Exception { + final String database = "root"; + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + try { + statement.execute(String.format("create database %s", database)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + TSStatusCode.ILLEGAL_PATH.getStatusCode() + + ": root is not a legal path, because the database name in tree model must start with 'root.'.", + e.getMessage()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateStorageGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateStorageGroupIT.java deleted file mode 100644 index 84dc7b72f474d..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateStorageGroupIT.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.it.schema; - -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.iotdb.util.AbstractSchemaIT; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runners.Parameterized; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.fail; - -/** - * Notice that, all test begins with "IoTDB" is integration test. All test which will start the - * IoTDB server should be defined as integration test. - */ -@Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IoTDBCreateStorageGroupIT extends AbstractSchemaIT { - - public IoTDBCreateStorageGroupIT(SchemaTestMode schemaTestMode) { - super(schemaTestMode); - } - - @Parameterized.BeforeParam - public static void before() throws Exception { - setUpEnvironment(); - EnvFactory.getEnv().initClusterEnvironment(); - } - - @Parameterized.AfterParam - public static void after() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - tearDownEnvironment(); - } - - @After - public void tearDown() throws Exception { - clearSchema(); - } - - /** The test creates three databases */ - @Test - public void testCreateStorageGroup() throws Exception { - String[] storageGroups = {"root.sg1", "root.sg2", "root.sg3"}; - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - for (String storageGroup : storageGroups) { - statement.execute(String.format("create database %s", storageGroup)); - } - - // ensure that current StorageGroup in cache is right. - createStorageGroupTool(statement, storageGroups); - } - // todo test restart - // EnvironmentUtils.stopDaemon(); - // setUp(); - // - // // ensure StorageGroup in cache is right after recovering. - // createStorageGroupTool(storageGroups); - } - - private void createStorageGroupTool(Statement statement, String[] storageGroups) - throws SQLException { - - List resultList = new ArrayList<>(); - try (ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { - while (resultSet.next()) { - String storageGroupPath = resultSet.getString(ColumnHeaderConstant.DATABASE); - resultList.add(storageGroupPath); - } - } - Assert.assertEquals(3, resultList.size()); - - resultList = resultList.stream().sorted().collect(Collectors.toList()); - - Assert.assertEquals(storageGroups[0], resultList.get(0)); - Assert.assertEquals(storageGroups[1], resultList.get(1)); - Assert.assertEquals(storageGroups[2], resultList.get(2)); - } - - /** Test creating a database that path is an existence database */ - @Test - public void testCreateExistStorageGroup1() throws Exception { - String storageGroup = "root.sg"; - - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute(String.format("CREATE DATABASE %s", storageGroup)); - - try { - statement.execute(String.format("create database %s", storageGroup)); - fail(); - } catch (SQLException e) { - Assert.assertEquals( - TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() - + ": root.sg has already been created as database", - e.getMessage()); - } - } - } - - /** Test the parent node has been set as a database */ - @Test - public void testCreateExistStorageGroup2() throws Exception { - - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute("create database root.sg"); - - try { - statement.execute("create database root.sg.`device`"); - fail(); - } catch (SQLException e) { - Assert.assertEquals( - TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() - + ": root.sg has already been created as database", - e.getMessage()); - } - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java index 9bb22ea599a42..ab16fb56248fa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java index 3defa886f8dbe..7af1fbfee3e42 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.util.AbstractSchemaIT; @@ -35,7 +35,6 @@ import java.sql.ResultSetMetaData; import java.sql.Statement; -import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.count; import static org.junit.Assert.fail; @@ -214,7 +213,7 @@ public void deleteTimeseriesAndChangeDeviceAlignmentTest() throws Exception { @Test public void deleteTimeSeriesMultiIntervalTest() { - String[] retArray1 = new String[] {"0,0"}; + String[] retArray1 = new String[] {"0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -232,10 +231,7 @@ public void deleteTimeSeriesMultiIntervalTest() { statement.executeQuery( "select count(s1) from root.sg.d1 where time >= 3 and time <= 4")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count("root.sg.d1.s1")); + String ans = resultSet.getString(count("root.sg.d1.s1")); Assert.assertEquals(retArray1[cnt], ans); cnt++; } @@ -246,4 +242,46 @@ public void deleteTimeSeriesMultiIntervalTest() { fail(e.getMessage()); } } + + @Test + public void deleteTimeseriesAndCreateSameTypeTest2() throws Exception { + String[] retArray = new String[] {"1,4.0,", "2,8.0,"}; + int cnt = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "create aligned timeseries root.turbine1.d1(s1 FLOAT encoding=PLAIN compression=SNAPPY, " + + "s2 INT64 encoding=PLAIN compression=SNAPPY, s4 DOUBLE encoding=PLAIN compression=SNAPPY)"); + statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1,s2,s4) ALIGNED VALUES(1,1,2,4)"); + + try (ResultSet resultSet = statement.executeQuery("SELECT s4 FROM root.turbine1.d1")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + Assert.assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + } + // delete series in the middle + statement.execute("DELETE timeseries root.turbine1.d1.s4"); + statement.execute( + "INSERT INTO root.turbine1.d1(timestamp,s3,s4) ALIGNED VALUES(2,false,8.0)"); + statement.execute("FLUSH"); + + try (ResultSet resultSet = statement.executeQuery("SELECT s4 FROM root.turbine1.d1")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + Assert.assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + } + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteStorageGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteStorageGroupIT.java index f99c84e2d5f7e..5315bd08c4d31 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteStorageGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteStorageGroupIT.java @@ -24,6 +24,7 @@ import org.apache.iotdb.util.AbstractSchemaIT; import org.junit.After; +import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized; @@ -113,8 +114,8 @@ public void testDeleteMultipleStorageGroupWithQuote() throws Exception { @Test(expected = SQLException.class) public void deleteNonExistStorageGroup() throws Exception { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE root.ln2.wf01.wt01"); statement.execute("DELETE DATABASE root.ln2.wf01.wt02"); } @@ -182,4 +183,24 @@ public void testDeleteStorageGroupAndThenQuery() throws Exception { assertEquals(1, count); } } + + @Test + public void testDeleteStorageGroupInvalidateCache() throws Exception { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + try { + statement.execute("insert into root.sg1.d1(s1) values(1);"); + statement.execute("insert into root.sg2(s2) values(1);"); + statement.execute("select last(s1) from root.sg1.d1;"); + statement.execute("select last(s2) from root.sg2;"); + statement.execute("insert into root.sg1.d1(s1) values(1);"); + statement.execute("insert into root.sg2(s2) values(1);"); + statement.execute("delete database root.**"); + statement.execute("insert into root.sg1.d1(s1) values(\"2001-08-01\");"); + statement.execute("insert into root.sg2(s2) values(\"2001-08-01\");"); + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java index b737dccf71f67..4d7a07dcebb87 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteTimeSeriesIT.java @@ -37,7 +37,6 @@ import java.sql.SQLException; import java.sql.Statement; -import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.constant.TestConstant.count; import static org.junit.Assert.fail; @@ -170,7 +169,7 @@ public void deleteTimeSeriesAndCreateSameTypeTest() throws Exception { @Test public void deleteTimeSeriesMultiIntervalTest() { - String[] retArray1 = new String[] {"0,0"}; + String[] retArray1 = new String[] {"0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -188,10 +187,7 @@ public void deleteTimeSeriesMultiIntervalTest() { statement.executeQuery( "select count(s1) from root.sg.d1 where time >= 3 and time <= 4")) { while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(count("root.sg.d1.s1")); + String ans = resultSet.getString(count("root.sg.d1.s1")); Assert.assertEquals(retArray1[cnt], ans); cnt++; } @@ -205,7 +201,7 @@ public void deleteTimeSeriesMultiIntervalTest() { @Test public void deleteTimeSeriesAndAutoDeleteDeviceTest() throws Exception { - String[] retArray1 = new String[] {"0,4,4,4,4"}; + String[] retArray1 = new String[] {"4,4,4,4"}; String insertSql = "insert into root.sg.d1(time, s1, s2, s3, s4) values(%d, %d, %d, %d, %d)"; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -219,8 +215,8 @@ public void deleteTimeSeriesAndAutoDeleteDeviceTest() throws Exception { statement.executeQuery( "select count(s1), count(s2), count(s3), count(s4) from root.sg.d1")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg.d1.s1"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg.d1.s" + i))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -244,9 +240,22 @@ public void deleteTimeSeriesAndAutoDeleteDeviceTest() throws Exception { } } + @Test + public void deleteTimeSeriesAndInvalidationTest() throws Exception { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("insert into root.sg.d1 (c1, c2) values (1, 1)"); + statement.execute("delete timeSeries root.sg.d1.**"); + try (final ResultSet resultSet = statement.executeQuery("select c1, c2 from root.sg.d1")) { + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + Assert.assertFalse(resultSet.next()); + } + } + } + @Test public void deleteTimeSeriesCrossSchemaRegionTest() throws Exception { - String[] retArray1 = new String[] {"0,4,4,4,4"}; + String[] retArray1 = new String[] {"4,4,4,4"}; String insertSql = "insert into root.sg.d%d(time, s1, s2) values(%d, %d, %d)"; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -260,8 +269,8 @@ public void deleteTimeSeriesCrossSchemaRegionTest() throws Exception { int cnt = 0; try (ResultSet resultSet = statement.executeQuery("select count(s1) from root.sg.*")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg.d1.s1"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg.d" + i + ".s1"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -282,8 +291,8 @@ public void deleteTimeSeriesCrossSchemaRegionTest() throws Exception { cnt = 0; try (ResultSet resultSet = statement.executeQuery("select count(s2) from root.sg.*")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg.d1.s2"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg.d" + i + ".s2"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -296,7 +305,7 @@ public void deleteTimeSeriesCrossSchemaRegionTest() throws Exception { @Test public void deleteTimeSeriesCrossStorageGroupTest() throws Exception { - String[] retArray1 = new String[] {"0,4,4,4,4"}; + String[] retArray1 = new String[] {"4,4,4,4"}; String insertSql = "insert into root.sg%d.d1(time, s1, s2) values(%d, %d, %d)"; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -310,8 +319,8 @@ public void deleteTimeSeriesCrossStorageGroupTest() throws Exception { int cnt = 0; try (ResultSet resultSet = statement.executeQuery("select count(s1) from root.*.d1")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg1.d1.s1"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg" + i + ".d1.s1"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -332,8 +341,8 @@ public void deleteTimeSeriesCrossStorageGroupTest() throws Exception { cnt = 0; try (ResultSet resultSet = statement.executeQuery("select count(s2) from root.*.*")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg1.d1.s2"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg" + i + ".d1.s2"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -351,12 +360,12 @@ public void deleteTimeSeriesCrossStorageGroupTest() throws Exception { Assert.assertFalse(resultSet.next()); } - retArray1 = new String[] {"0,4,4"}; + retArray1 = new String[] {"4,4"}; cnt = 0; try (ResultSet resultSet = statement.executeQuery("select count(s2) from root.*.*")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 3; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg3.d1.s2"))); + for (int i = 4; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg" + i + ".d1.s2"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -369,7 +378,7 @@ public void deleteTimeSeriesCrossStorageGroupTest() throws Exception { @Test public void deleteTimeSeriesWithMultiPatternTest() throws Exception { - String[] retArray1 = new String[] {"0,4,4,4,4"}; + String[] retArray1 = new String[] {"4,4,4,4"}; String insertSql = "insert into root.sg%d.d1(time, s1, s2) values(%d, %d, %d)"; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -383,8 +392,8 @@ public void deleteTimeSeriesWithMultiPatternTest() throws Exception { int cnt = 0; try (ResultSet resultSet = statement.executeQuery("select count(s1) from root.*.d1")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg1.d1.s1"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg" + i + ".d1.s1"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); @@ -450,7 +459,7 @@ public void deleteTimeSeriesAndReturnPathNotExistsTest() throws Exception { + ": Timeseries [root.**] does not exist or is represented by device template")); } - String[] retArray1 = new String[] {"0,4,4,4,4"}; + String[] retArray1 = new String[] {"4,4,4,4"}; String insertSql = "insert into root.sg%d.d1(time, s1, s2) values(%d, %d, %d)"; for (int i = 1; i <= 4; i++) { @@ -463,8 +472,8 @@ public void deleteTimeSeriesAndReturnPathNotExistsTest() throws Exception { try (ResultSet resultSet = statement.executeQuery("select count(s1) from root.*.d1")) { while (resultSet.next()) { - StringBuilder ans = new StringBuilder(resultSet.getString(TIMESTAMP_STR)); - for (int i = 1; i <= 4; i++) { + StringBuilder ans = new StringBuilder(resultSet.getString(count("root.sg1.d1.s1"))); + for (int i = 2; i <= 4; i++) { ans.append(",").append(resultSet.getString(count("root.sg" + i + ".d1.s1"))); } Assert.assertEquals(retArray1[cnt], ans.toString()); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java index 9945843b09185..f3d53c3a46187 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java @@ -142,10 +142,15 @@ public void testAutoExtendTemplate() throws SQLException { statement.execute("SET DEVICE TEMPLATE t1 to root.db"); + // single-row insertion statement.execute("INSERT INTO root.db.d1(time, s1, s3) values(1, 1, 1)"); statement.execute("INSERT INTO root.db.d2(time, s4, s5) values(1, 1, 1)"); statement.execute("INSERT INTO root.db1.d1(time, s2, s3) values(1, 1, 1)"); + // multi-row insertion with null + statement.execute( + "INSERT INTO root.db.d1(time, s1, s6) values(1, 1, 1), (2, 2, null), (3, 3, 3)"); + String[] sqls = new String[] { "show timeseries", @@ -159,11 +164,13 @@ public void testAutoExtendTemplate() throws SQLException { "root.db.d1.s3,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db.d1.s4,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db.d1.s5,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", + "root.db.d1.s6,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db.d2.s1,null,root.db,INT64,PLAIN,LZ4,null,null,null,null,BASE,", "root.db.d2.s2,null,root.db,DOUBLE,RLE,LZ4,null,null,null,null,BASE,", "root.db.d2.s3,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db.d2.s4,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db.d2.s5,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", + "root.db.d2.s6,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db1.d1.s2,null,root.db1,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", "root.db1.d1.s3,null,root.db1,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,")) }; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java index ecc16af137cc3..1f4445ddb46e0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaTemplateIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSortedShowTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSortedShowTimeseriesIT.java index d68e768440772..8c70e76c1fcfa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSortedShowTimeseriesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSortedShowTimeseriesIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagAlterIT.java index d4afa4e3d04a7..feb49793c0868 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagAlterIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -611,28 +611,28 @@ public void upsertTest() { @Test public void alterDuplicateAliasTest() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute( "create timeseries root.turbine.d1.s1(a1) with datatype=FLOAT, encoding=RLE, compression=SNAPPY;"); statement.execute("create timeseries root.turbine.d1.s2 with datatype=INT32, encoding=RLE;"); try { statement.execute("alter timeseries root.turbine.d1.s2 upsert alias=s1;"); fail(); - } catch (Exception e) { + } catch (final Exception e) { assertTrue( e.getMessage() - .contains("The alias is duplicated with the name or alias of other measurement.")); + .contains("The alias is duplicated with the name or alias of other measurement")); } try { statement.execute("alter timeseries root.turbine.d1.s2 upsert alias=a1;"); fail(); - } catch (Exception e) { + } catch (final Exception e) { assertTrue( e.getMessage() - .contains("The alias is duplicated with the name or alias of other measurement.")); + .contains("The alias is duplicated with the name or alias of other measurement")); } - } catch (Exception e) { + } catch (final Exception e) { e.printStackTrace(); fail(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java index 0fc239067d88f..2dd80c2bba0fe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagLimitIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagLimitIT.java index aea1e5141514e..3e2df365d5648 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagLimitIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBTagLimitIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.schema; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java index 90c5ea657254c..b92176d182de6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java @@ -570,6 +570,27 @@ public void testExpressionAlignByDevice2() { queryRetArray); } + @Test + public void testAliasAlignByDevice() { + String[] intoRetArray = + new String[] { + "root.sg.d1,s1,root.sg_abd_alias.d1.k1,10,", + }; + resultSetEqualTest( + "select s1 as k1 " + "into root.sg_abd_alias.d1(::) from root.sg.d1 align by device;", + selectIntoAlignByDeviceHeader, + intoRetArray); + + intoRetArray = + new String[] { + "k1,root.sg_abd_alias.d2.k1,10,", + }; + resultSetEqualTest( + "select s1 as k1 " + "into root.sg_abd_alias.d2(::) from root.sg.d1;", + selectIntoHeader, + intoRetArray); + } + // -------------------------------------- CHECK EXCEPTION ------------------------------------- @Test @@ -778,11 +799,60 @@ public void testNewDataType() { String[] resultSet = new String[] { - "1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", - "3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + "1,1997-01-01T00:00:00.001Z,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "3,1997-01-03T00:00:00.001Z,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", }; String expectedQueryHeader = "Time,root.db.d2.s7,root.db.d2.s8,root.db.d2.s9,root.db.d2.s10,"; resultSetEqualTest("select s7,s8,s9,s10 from root.db.d2;", expectedQueryHeader, resultSet); } + + // -------------------------------------- OTHER TEST ------------------------------------- + @Test + public void testRemoveBackQuote() { + String[] intoRetArray = + new String[] { + "count(root.sg.d1.s1),root.sg_agg1.d1.count_s1,1,", + "last_value(root.sg.d1.s2),root.sg_agg1.d1.last_value_s2,1,", + "count(root.sg.d2.s1),root.sg_agg1.d2.count_s1,1,", + "last_value(root.sg.d2.s2),root.sg_agg1.d2.last_value_s2,1," + }; + resultSetEqualTest( + "select count(d1.s1), last_value(d1.s2), count(d2.s1), last_value(d2.s2) " + + "into root.sg_agg1.`d1`(`count_s1`, last_value_s2), aligned root.sg_agg1.d2(count_s1, last_value_s2) " + + "from root.sg;", + selectIntoHeader, + intoRetArray); + + String expectedQueryHeader = + "Time,root.sg_agg1.d1.count_s1,root.sg_agg1.d2.count_s1,root.sg_agg1.d1.last_value_s2,root.sg_agg1.d2.last_value_s2,"; + String[] queryRetArray = new String[] {"0,10,7,12.0,11.0,"}; + resultSetEqualTest( + "select count_s1, last_value_s2 from root.sg_agg1.d1, root.sg_agg1.d2;", + expectedQueryHeader, + queryRetArray); + } + + @Test + public void testRemoveBackQuoteAlignByDevice() { + String[] intoRetArray = + new String[] { + "root.sg.d1,count(s1),root.sg_abd_agg1.d1.count_s1,1,", + "root.sg.d1,last_value(s2),root.sg_abd_agg1.d1.last_value_s2,1," + }; + resultSetEqualTest( + "select count(s1), last_value(s2) " + + "into root.sg_abd_agg1.`d1`(`count_s1`, last_value_s2) " + + "from root.sg.d1 align by device;", + selectIntoAlignByDeviceHeader, + intoRetArray); + + String expectedQueryHeader = + "Time,root.sg_abd_agg1.d1.count_s1," + "root.sg_abd_agg1.d1.last_value_s2,"; + String[] queryRetArray = new String[] {"0,10,12.0,"}; + resultSetEqualTest( + "select count_s1, last_value_s2 from root.sg_abd_agg1.d1, root.sg_abd_agg1.d2;", + expectedQueryHeader, + queryRetArray); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/strangepath/IoTDBStrangePathIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/strangepath/IoTDBStrangePathIT.java index 8e022abf93fc1..ad60775936cf3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/strangepath/IoTDBStrangePathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/strangepath/IoTDBStrangePathIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.strangepath; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java index cb21faa628d3f..b671020855640 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.trigger; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -286,6 +286,53 @@ public void testCreateTriggersNormally() { } } + @Test + public void testCreateTriggersNormally2() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + Map result = + new HashMap() { + { + put( + STATELESS_TRIGGER_BEFORE_INSERTION_PREFIX + "a", + new String[] { + STATELESS_TRIGGER_BEFORE_INSERTION_PREFIX + "a", + BEFORE_INSERT, + STATELESS, + ACTIVE, + "root.test.stateless.a", + TRIGGER_FILE_TIMES_COUNTER + }); + } + }; + + // create stateless triggers before insertion + statement.execute( + String.format( + "create stateless trigger %s before insert on root.test.stateless.a as '%s' with (\"name\"=\"%s\")", + STATELESS_TRIGGER_BEFORE_INSERTION_PREFIX + "a", + TRIGGER_FILE_TIMES_COUNTER, + STATELESS_TRIGGER_BEFORE_INSERTION_PREFIX + "a")); + + ResultSet resultSet = statement.executeQuery("show triggers"); + int cnt = 0; + while (resultSet.next()) { + cnt++; + String triggerName = resultSet.getString(ColumnHeaderConstant.TRIGGER_NAME); + String[] triggerInformation = result.get(triggerName); + assertEquals(triggerInformation[0], triggerName); + assertEquals(triggerInformation[1], resultSet.getString(ColumnHeaderConstant.EVENT)); + assertEquals(triggerInformation[2], resultSet.getString(ColumnHeaderConstant.TYPE)); + assertEquals(triggerInformation[3], resultSet.getString(ColumnHeaderConstant.STATE)); + assertEquals(triggerInformation[4], resultSet.getString(ColumnHeaderConstant.PATH_PATTERN)); + assertEquals(triggerInformation[5], resultSet.getString(ColumnHeaderConstant.CLASS_NAME)); + } + assertEquals(cnt, result.size()); + } catch (Exception e) { + fail(e.getMessage()); + } + } + @Test public void testCreateAndDropMultipleTimes() { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFGroupByLevelIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFGroupByLevelIT.java index fc5833a292433..35447f295747f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFGroupByLevelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFGroupByLevelIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.it.udaf; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java index 75dc0c1f5f770..12d8c1ac34808 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java @@ -81,7 +81,7 @@ public void createReflectShowDropUDAFTest() { statement.executeQuery("SELECT udaf(*) FROM root.vehicle"); try (ResultSet resultSet = statement.executeQuery("SHOW FUNCTIONS")) { - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); int count = 0; while (resultSet.next()) { ++count; @@ -108,7 +108,7 @@ public void createAndDropUDAFSeveralTimesTest() { ++count; } Assert.assertEquals(1 + FUNCTIONS_COUNT, count); - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); statement.execute("DROP FUNCTION udaf"); statement.execute( @@ -122,7 +122,7 @@ public void createAndDropUDAFSeveralTimesTest() { ++count; } Assert.assertEquals(1 + FUNCTIONS_COUNT, count); - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); statement.execute("DROP FUNCTION udaf"); } } catch (SQLException throwable) { @@ -222,7 +222,7 @@ public void createFunctionWithURITest() throws SQLException { ++count; } Assert.assertEquals(2 + FUNCTIONS_COUNT, count); - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); statement.execute("DROP FUNCTION udaf1"); statement.execute("DROP FUNCTION udaf2"); } catch (Exception e) { @@ -309,7 +309,7 @@ public void testShowBuiltinFunction() { "CREATE FUNCTION udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFCount'"); try (ResultSet resultSet = statement.executeQuery("SHOW FUNCTIONS")) { - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); int count = 0; while (resultSet.next()) { StringBuilder stringBuilder = new StringBuilder(); @@ -320,7 +320,7 @@ public void testShowBuiltinFunction() { if (result.contains(FUNCTION_TYPE_EXTERNAL_UDAF)) { Assert.assertEquals( String.format( - "UDAF,%s,org.apache.iotdb.db.query.udf.example.UDAFCount,", + "UDAF,%s,org.apache.iotdb.db.query.udf.example.UDAFCount,AVAILABLE,", FUNCTION_TYPE_EXTERNAL_UDAF), result); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java index 56c7a7f4e3793..82f61879dd774 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java @@ -45,7 +45,6 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBUDAFNormalQueryIT { private static final double DELTA = 1E-6; - protected static final String TIMESTAMP_STR = "Time"; private static final String[] creationSqls = new String[] { @@ -198,7 +197,7 @@ private static void registerUDAF() { @Test public void singleUDAFTest() { - String[] retArray = new String[] {"0,2001,2001,2001,2001", "0,7500,7500,7500,7500"}; + String[] retArray = new String[] {"2001,2001,2001,2001", "7500,7500,7500,7500"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -210,9 +209,7 @@ public void singleUDAFTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(countUDAF(d0s0)) + resultSet.getString(countUDAF(d0s0)) + "," + resultSet.getString(countUDAF(d0s1)) + "," @@ -231,9 +228,7 @@ public void singleUDAFTest() { + "FROM root.vehicle.d0")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(countUDAF(d0s0)) + resultSet.getString(countUDAF(d0s0)) + "," + resultSet.getString(countUDAF(d0s1)) + "," @@ -254,9 +249,7 @@ public void singleUDAFTest() { cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(countUDAF(d0s0)) + resultSet.getString(countUDAF(d0s0)) + "," + resultSet.getString(countUDAF(d0s1)) + "," @@ -275,9 +268,7 @@ public void singleUDAFTest() { + "FROM root.vehicle.d0 order by time desc")) { while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString(countUDAF(d0s0)) + resultSet.getString(countUDAF(d0s0)) + "," + resultSet.getString(countUDAF(d0s1)) + "," @@ -298,8 +289,8 @@ public void singleUDAFTest() { @Test public void multipleUDAFTest() { double[][] retArray = { - {0.0, 1.4508E7, 7250.374812593702}, - {0.0, 626750.0, 1250.9980039920158} + {1.4508E7, 7250.374812593702}, + {626750.0, 1250.9980039920158} }; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -311,10 +302,9 @@ public void multipleUDAFTest() { "SELECT sum_udaf(s0),avg_udaf(s2)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000")) { while (resultSet.next()) { - double[] ans = new double[3]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sumUDAF(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avgUDAF(d0s2))); + double[] ans = new double[2]; + ans[0] = Double.parseDouble(resultSet.getString(sumUDAF(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avgUDAF(d0s2))); assertArrayEquals(retArray[cnt], ans, DELTA); cnt++; } @@ -326,10 +316,9 @@ public void multipleUDAFTest() { "SELECT sum_udaf(s0),avg_udaf(s2)" + "FROM root.vehicle.d0 WHERE time >= 1000 AND time <= 2000")) { while (resultSet.next()) { - double[] ans = new double[3]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sumUDAF(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avgUDAF(d0s2))); + double[] ans = new double[2]; + ans[0] = Double.parseDouble(resultSet.getString(sumUDAF(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avgUDAF(d0s2))); assertArrayEquals(retArray[cnt], ans, DELTA); cnt++; } @@ -343,10 +332,9 @@ public void multipleUDAFTest() { "SELECT sum_udaf(s0),avg_udaf(s2)" + "FROM root.vehicle.d0 WHERE time >= 6000 AND time <= 9000 order by time desc")) { while (resultSet.next()) { - double[] ans = new double[3]; - ans[0] = Double.parseDouble(resultSet.getString(TIMESTAMP_STR)); - ans[1] = Double.parseDouble(resultSet.getString(sumUDAF(d0s0))); - ans[2] = Double.parseDouble(resultSet.getString(avgUDAF(d0s2))); + double[] ans = new double[2]; + ans[0] = Double.parseDouble(resultSet.getString(sumUDAF(d0s0))); + ans[1] = Double.parseDouble(resultSet.getString(avgUDAF(d0s2))); assertArrayEquals(retArray[cnt], ans, DELTA); cnt++; } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java index 5de421ebd2a6d..71555bce0e3cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java @@ -152,4 +152,16 @@ public void testUDFSingleRowQuery() { fail(throwable.getMessage()); } } + + @Test + public void testUntrustedUri() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "CREATE FUNCTION two_sum AS 'org.apache.iotdb.db.query.udf.example.TwoSum' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'"); + fail("should fail"); + } catch (SQLException throwable) { + assertTrue(throwable.getMessage().contains("701: Untrusted uri ")); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java index 798256e5a16d6..137b5e05ab027 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.udf; import org.apache.iotdb.it.env.EnvFactory; @@ -87,7 +88,7 @@ public void testCreateReflectShowDrop() { statement.executeQuery("select udf(*, *) from root.vehicle"); try (ResultSet resultSet = statement.executeQuery("show functions")) { - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); int count = 0; while (resultSet.next()) { StringBuilder stringBuilder = new StringBuilder(); @@ -123,7 +124,7 @@ public void testCreateAndDropSeveralTimes() { Assert.assertEquals( 1 + NATIVE_FUNCTIONS_COUNT + BUILTIN_FUNCTIONS_COUNT + BUILTIN_SCALAR_FUNCTIONS_COUNT, count); - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); statement.execute("drop function udf"); statement.execute("create function udf as 'org.apache.iotdb.db.query.udf.example.Adder'"); @@ -138,7 +139,7 @@ public void testCreateAndDropSeveralTimes() { Assert.assertEquals( 1 + NATIVE_FUNCTIONS_COUNT + BUILTIN_FUNCTIONS_COUNT + BUILTIN_SCALAR_FUNCTIONS_COUNT, count); - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); statement.execute("drop function udf"); } } catch (SQLException throwable) { @@ -249,7 +250,7 @@ public void testCreateFunctionWithURI() throws SQLException { Assert.assertEquals( 2 + NATIVE_FUNCTIONS_COUNT + BUILTIN_FUNCTIONS_COUNT + BUILTIN_SCALAR_FUNCTIONS_COUNT, count); - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); statement.execute("drop function udf"); statement.execute("drop function udf1"); } catch (Exception e) { @@ -353,7 +354,10 @@ public void testCreateBuiltinFunction() { statement.execute("create function sin as 'org.apache.iotdb.db.query.udf.example.Adder'"); fail(); } catch (SQLException throwable) { - assertTrue(throwable.getMessage().contains("the same name UDF has been created")); + assertTrue( + throwable + .getMessage() + .contains("the given function name conflicts with the built-in function name")); } } @@ -376,7 +380,7 @@ public void testShowBuiltinFunction() { statement.execute("create function udf as 'org.apache.iotdb.db.query.udf.example.Adder'"); try (ResultSet resultSet = statement.executeQuery("show functions")) { - assertEquals(3, resultSet.getMetaData().getColumnCount()); + assertEquals(4, resultSet.getMetaData().getColumnCount()); int count = 0; while (resultSet.next()) { StringBuilder stringBuilder = new StringBuilder(); @@ -391,7 +395,7 @@ public void testShowBuiltinFunction() { if (result.contains(FUNCTION_TYPE_EXTERNAL_UDTF)) { Assert.assertEquals( String.format( - "UDF,%s,org.apache.iotdb.db.query.udf.example.Adder,", + "UDF,%s,org.apache.iotdb.db.query.udf.example.Adder,AVAILABLE,", FUNCTION_TYPE_EXTERNAL_UDTF), result); ++count; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java index 36de61e44118b..c543d4e6bd10a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.udf; import org.apache.iotdb.it.env.EnvFactory; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java index 24aae4065351e..e3955df58d604 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.udf; import org.apache.iotdb.it.env.EnvFactory; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java index 2ddc99f6ead58..61de72829ccc9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java @@ -205,7 +205,7 @@ public void testSelectorFunctions() { } else if (j == 5) { assertEquals("2024-01-0" + (i + 1), resultSet.getString(7)); } else if (j == 6) { - assertEquals(String.valueOf(i), resultSet.getString(8)); + assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8)); } } } @@ -251,7 +251,7 @@ public void testSelectorFunctions() { } else if (j == 5) { assertEquals("2024-01-0" + (i + 1), resultSet.getString(7)); } else if (j == 6) { - assertEquals(String.valueOf(i), resultSet.getString(8)); + assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8)); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java index 09850109297b1..93e7a09bb51af 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.udf; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -124,7 +125,7 @@ public void testUserDefinedBuiltInHybridAggregationQuery() { @Test public void testUserDefinedBuiltInHybridAggregationQuery2() { - String[] retArray = new String[] {"0,2.0,0.9092974268256817,3.0,-10.0,12.0"}; + String[] retArray = new String[] {"2.0,0.9092974268256817,3.0,-10.0,12.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -134,9 +135,7 @@ public void testUserDefinedBuiltInHybridAggregationQuery2() { int cnt = 0; while (resultSet.next()) { String ans = - resultSet.getString(TIMESTAMP_STR) - + "," - + resultSet.getString("avg(root.vehicle.d1.s1)") + resultSet.getString("avg(root.vehicle.d1.s1)") + "," + resultSet.getString("sin(avg(root.vehicle.d1.s2))") + "," diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java index dbf9dfd3e589f..386ff4c159089 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.it.udf; import org.apache.iotdb.it.env.EnvFactory; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 372c9200c5e6c..ec98e4dcb98db 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -27,6 +27,7 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.tsfile.read.common.RowRecord; @@ -40,18 +41,23 @@ import java.sql.SQLException; import java.sql.Statement; import java.text.DateFormat; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import static org.apache.iotdb.itbase.constant.TestConstant.DELTA; import static org.apache.iotdb.itbase.constant.TestConstant.NULL; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; +import static org.apache.iotdb.itbase.env.BaseEnv.TREE_SQL_DIALECT; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -63,6 +69,19 @@ public class TestUtils { private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); + public static final ZoneId DEFAULT_ZONE_ID = ZoneId.ofOffset("UTC", ZoneOffset.of("Z")); + + public static final String TIME_PRECISION_IN_MS = "ms"; + + public static final String TIME_PRECISION_IN_US = "us"; + + public static final String TIME_PRECISION_IN_NS = "ns"; + + public static String defaultFormatDataTime(long time) { + return RpcUtils.formatDatetime( + RpcUtils.DEFAULT_TIME_FORMAT, TIME_PRECISION_IN_MS, time, DEFAULT_ZONE_ID); + } + public static void prepareData(String[] sqls) { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -89,6 +108,32 @@ public static void prepareData(List sqls) { } } + public static void prepareTableData(String[] sqls) { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void prepareTableData(List sqls) { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + public static void resultSetEqualTest(String sql, double[][] retArray, String[] columnNames) { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -174,6 +219,196 @@ public static void resultSetEqualTest( resultSetEqualTest(sql, expectedHeader, expectedRetArray, null); } + public static void tableResultSetEqualTest( + String sql, String[] expectedHeader, String[] expectedRetArray, String database) { + tableResultSetEqualTest( + sql, + expectedHeader, + expectedRetArray, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + database); + } + + public static void tableResultSetEqualTest( + String sql, + String[] expectedHeader, + String[] expectedRetArray, + String userName, + String password, + String database) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= expectedHeader.length; i++) { + builder.append(resultSet.getString(i)).append(","); + } + assertEquals(expectedRetArray[cnt], builder.toString()); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableExecuteTest(String sql, String userName, String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableQueryNoVerifyResultTest( + String sql, String[] expectedHeader, String userName, String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableResultSetEqualTest( + String sql, + String[] expectedHeader, + String[] expectedRetArray, + String userName, + String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= expectedHeader.length; i++) { + builder.append(resultSet.getString(i)).append(","); + } + assertEquals(expectedRetArray[cnt], builder.toString()); + // System.out.println(String.format("\"%s\",", builder.toString())); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableResultSetFuzzyTest( + String sql, String[] expectedHeader, int expectedCount, String database) { + tableResultSetFuzzyTest( + sql, + expectedHeader, + expectedCount, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + database); + } + + public static void tableResultSetFuzzyTest( + String sql, + String[] expectedHeader, + int expectedCount, + String userName, + String password, + String database) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + assertEquals(expectedCount, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableAssertTestFail(String sql, String errMsg, String databaseName) { + tableAssertTestFail( + sql, errMsg, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, databaseName); + } + + public static void tableAssertTestFail( + String sql, String errMsg, String userName, String password, String databaseName) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + databaseName); + statement.executeQuery(sql); + fail("No exception!"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + + public static void tableAssertTestFail( + String sql, String errMsg, String userName, String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.executeQuery(sql); + fail("No exception!"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + public static void resultSetEqualTest( String sql, String[] expectedHeader, @@ -300,15 +535,45 @@ public static void assertTestFail( } } + public static void assertTableTestFail( + final BaseEnv env, + final String sql, + final String errMsg, + final String userName, + final String password, + final String db) { + try (final Connection connection = + env.getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + if (Objects.nonNull(db)) { + statement.execute("use " + "\"" + db + "\""); + } + statement.executeQuery(sql); + fail("No exception!"); + } catch (final SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + public static void assertNonQueryTestFail(String sql, String errMsg) { assertNonQueryTestFail(sql, errMsg, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); } + public static void assertTableNonQueryTestFail(String sql, String errMsg, String dbName) { + assertTableNonQueryTestFail( + sql, errMsg, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, dbName); + } + public static void assertNonQueryTestFail( String sql, String errMsg, String userName, String password) { assertNonQueryTestFail(EnvFactory.getEnv(), sql, errMsg, userName, password); } + public static void assertTableNonQueryTestFail( + String sql, String errMsg, String userName, String password, String dbName) { + assertTableNonQueryTestFail(EnvFactory.getEnv(), sql, errMsg, userName, password, dbName); + } + public static void assertNonQueryTestFail( BaseEnv env, String sql, String errMsg, String userName, String password) { try (Connection connection = env.getConnection(userName, password); @@ -320,6 +585,33 @@ public static void assertNonQueryTestFail( } } + public static void assertTableNonQueryTestFail( + BaseEnv env, String sql, String errMsg, String userName, String password, String db) { + try (Connection connection = env.getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + if (Objects.nonNull(db)) { + statement.execute("use " + "\"" + db + "\""); + } + statement.execute(sql); + fail("No exception!"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + + public static void assertResultSetSize(final ResultSet actualResultSet, final int size) { + try { + int count = 0; + while (actualResultSet.next()) { + ++count; + } + Assert.assertEquals(size, count); + } catch (final Exception e) { + e.printStackTrace(); + Assert.fail(String.valueOf(e)); + } + } + public static void assertResultSetEqual( ResultSet actualResultSet, String expectedHeader, String[] expectedRetArray) { assertResultSetEqual( @@ -352,6 +644,35 @@ public static void assertResultSetEqual( } } + public static void assertResultSetEqual( + ResultSet actualResultSet, + String expectedHeader, + Set expectedRetSet, + Consumer consumer) { + try { + ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); + StringBuilder header = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + header.append(resultSetMetaData.getColumnName(i)).append(","); + } + assertEquals(expectedHeader, header.toString()); + + Set actualRetSet = new HashSet<>(); + + while (actualResultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(actualResultSet.getString(i)).append(","); + } + actualRetSet.add(builder.toString()); + } + assertEquals(expectedRetSet, actualRetSet); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(String.valueOf(e)); + } + } + public static void assertSingleResultSetEqual( ResultSet actualResultSet, Map expectedHeaderWithResult) { try { @@ -427,13 +748,57 @@ public static void executeNonQuery(String sql, String userName, String password) } } - public static void executeNonQueryWithRetry(BaseEnv env, String sql) { + public static void executeNonQueryWithRetry(final BaseEnv env, final String sql) { + executeNonQueryWithRetry(env, sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + } + + public static void executeNonQueryWithRetry( + final BaseEnv env, final String sql, final String userName, final String password) { + executeNonQueryWithRetry(env, sql, userName, password, null, TREE_SQL_DIALECT); + } + + public static void executeNonQueryWithRetry( + final BaseEnv env, + final String sql, + final String userName, + final String password, + final String database, + final String sqlDialect) { + executeNonQueriesWithRetry( + env, Collections.singletonList(sql), userName, password, database, sqlDialect); + } + + public static void executeNonQueriesWithRetry( + final BaseEnv env, final List sqlList, final String userName, final String password) { + executeNonQueriesWithRetry(env, sqlList, userName, password, "", TREE_SQL_DIALECT); + } + + public static void executeNonQueriesWithRetry( + final BaseEnv env, + final List sqlList, + final String userName, + final String password, + final String database, + final String sqlDialect) { + int lastIndex = 0; for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try (Connection connection = env.getConnection(); - Statement statement = connection.createStatement()) { - statement.execute(sql); - break; - } catch (SQLException e) { + try (final Connection connection = + env.getConnection( + userName, + password, + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) + ? BaseEnv.TABLE_SQL_DIALECT + : TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && database != null) { + statement.execute("use " + database); + } + for (int i = lastIndex; i < sqlList.size(); ++i) { + lastIndex = i; + statement.execute(sqlList.get(i)); + } + return; + } catch (final SQLException e) { if (retryCountLeft > 0) { try { Thread.sleep(10000); @@ -452,24 +817,81 @@ public static boolean tryExecuteNonQueryWithRetry(BaseEnv env, String sql) { env, sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); } + public static boolean tryExecuteNonQueryWithRetry( + String dataBaseName, String sqlDialect, BaseEnv env, String sql) { + return tryExecuteNonQueryWithRetry( + env, + sql, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + dataBaseName, + sqlDialect); + } + public static boolean tryExecuteNonQueryWithRetry( BaseEnv env, String sql, String userName, String password) { return tryExecuteNonQueriesWithRetry(env, Collections.singletonList(sql), userName, password); } + public static boolean tryExecuteNonQueryWithRetry( + BaseEnv env, + String sql, + String userName, + String password, + String dataBaseName, + String sqlDialect) { + return tryExecuteNonQueriesWithRetry( + env, Collections.singletonList(sql), userName, password, dataBaseName, sqlDialect); + } + public static boolean tryExecuteNonQueriesWithRetry(BaseEnv env, List sqlList) { return tryExecuteNonQueriesWithRetry( - env, sqlList, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + env, + sqlList, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + null, + TREE_SQL_DIALECT); + } + + public static boolean tryExecuteNonQueriesWithRetry( + String dataBase, String sqlDialect, BaseEnv env, List sqlList) { + return tryExecuteNonQueriesWithRetry( + env, + sqlList, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + dataBase, + sqlDialect); } // This method will not throw failure given that a failure is encountered. // Instead, it returns a flag to indicate the result of the execution. public static boolean tryExecuteNonQueriesWithRetry( BaseEnv env, List sqlList, String userName, String password) { + return tryExecuteNonQueriesWithRetry(env, sqlList, userName, password, null, TREE_SQL_DIALECT); + } + + public static boolean tryExecuteNonQueriesWithRetry( + BaseEnv env, + List sqlList, + String userName, + String password, + String dataBase, + String sqlDialect) { int lastIndex = 0; for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try (Connection connection = env.getConnection(userName, password); + try (Connection connection = + env.getConnection( + userName, + password, + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) + ? BaseEnv.TABLE_SQL_DIALECT + : TREE_SQL_DIALECT); Statement statement = connection.createStatement()) { + if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && dataBase != null) { + statement.execute("use " + dataBase); + } for (int i = lastIndex; i < sqlList.size(); ++i) { lastIndex = i; statement.execute(sqlList.get(i)); @@ -545,6 +967,42 @@ public static boolean tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( return false; } + public static boolean tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( + BaseEnv env, + DataNodeWrapper wrapper, + List sqlList, + String dataBase, + String sqlDialect) { + int lastIndex = 0; + for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { + try (Connection connection = + env.getWriteOnlyConnectionWithSpecifiedDataNode(wrapper, sqlDialect); + Statement statement = connection.createStatement()) { + + if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && dataBase != null) { + statement.execute("use " + dataBase); + } + + for (int i = lastIndex; i < sqlList.size(); ++i) { + statement.execute(sqlList.get(i)); + lastIndex = i; + } + return true; + } catch (SQLException e) { + if (retryCountLeft > 0) { + try { + Thread.sleep(10000); + } catch (InterruptedException ignored) { + } + } else { + e.printStackTrace(); + return false; + } + } + } + return false; + } + public static void executeQuery(String sql) { executeQuery(sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); } @@ -560,27 +1018,57 @@ public static void executeQuery(String sql, String userName, String password) { } public static void executeQueryWithRetry( - BaseEnv env, String sql, String userName, String password) { - try (Connection connection = env.getConnection(userName, password); - Statement statement = connection.createStatement()) { - for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try { - statement.executeQuery(sql); - } catch (SQLException e) { - if (retryCountLeft > 0) { - try { - Thread.sleep(10000); - } catch (InterruptedException ignored) { - } - } else { - e.printStackTrace(); - fail(e.getMessage()); + final BaseEnv env, final String sql, final String userName, final String password) { + executeQueryWithRetry(env, sql, userName, password, null, TREE_SQL_DIALECT); + } + + public static void executeQueryWithRetry( + final BaseEnv env, + final String sql, + final String userName, + final String password, + final String database, + final String sqlDialect) { + executeQueriesWithRetry( + env, Collections.singletonList(sql), userName, password, database, sqlDialect); + } + + public static void executeQueriesWithRetry( + final BaseEnv env, + final List sqlList, + final String userName, + final String password, + final String database, + final String sqlDialect) { + int lastIndex = 0; + for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { + try (final Connection connection = + env.getConnection( + userName, + password, + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) + ? BaseEnv.TABLE_SQL_DIALECT + : TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && database != null) { + statement.execute("use " + database); + } + for (int i = lastIndex; i < sqlList.size(); ++i) { + lastIndex = i; + statement.executeQuery(sqlList.get(i)); + } + return; + } catch (final SQLException e) { + if (retryCountLeft > 0) { + try { + Thread.sleep(10000); + } catch (InterruptedException ignored) { } + } else { + e.printStackTrace(); + fail(e.getMessage()); } } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); } } @@ -694,12 +1182,23 @@ public static void assertDataEventuallyOnEnv( assertDataEventuallyOnEnv(env, sql, expectedHeader, expectedResSet, 600); } + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final Consumer handleFailure) { + assertDataEventuallyOnEnv(env, sql, expectedHeader, expectedResSet, 600, handleFailure); + } + public static void assertDataEventuallyOnEnv( BaseEnv env, String sql, String expectedHeader, Set expectedResSet, long timeoutSeconds) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; try (Connection connection = env.getConnection(); Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures @@ -711,6 +1210,13 @@ public static void assertDataEventuallyOnEnv( .untilAsserted( () -> { try { + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] && System.currentTimeMillis() - startTime > timeoutSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } TestUtils.assertResultSetEqual( executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); } catch (Exception e) { @@ -723,6 +1229,287 @@ public static void assertDataEventuallyOnEnv( } } + public static void assertDataEventuallyOnEnv( + BaseEnv env, + DataNodeWrapper dataNodeWrapper, + String sql, + String expectedHeader, + Set expectedResSet, + long timeoutSeconds) { + try (Connection connection = + env.getConnection( + dataNodeWrapper, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + TestUtils.assertResultSetEqual( + executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); + } catch (Exception e) { + Assert.fail(); + } + }); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + public static void assertDataSizeEventuallyOnEnv( + final BaseEnv env, final String sql, final int size, final String databaseName) { + assertDataSizeEventuallyOnEnv(env, sql, size, 600, databaseName); + } + + public static void assertDataSizeEventuallyOnEnv( + final BaseEnv env, + final String sql, + final int size, + final long timeoutSeconds, + final String dataBaseName) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + if (dataBaseName != null) { + statement.execute("use " + dataBaseName); + } + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] && System.currentTimeMillis() - startTime > timeoutSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } + if (sql != null && !sql.isEmpty()) { + TestUtils.assertResultSetSize(executeQueryWithRetry(statement, sql), size); + } + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } + }); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final long timeoutSeconds, + final Consumer handleFailure) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; + try (Connection connection = env.getConnection(); + Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] && System.currentTimeMillis() - startTime > timeoutSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } + TestUtils.assertResultSetEqual( + executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); + } catch (Exception e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + Assert.fail(); + } catch (Error e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + throw e; + } + }); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final String dataBaseName) { + assertDataEventuallyOnEnv(env, sql, expectedHeader, expectedResSet, 600, dataBaseName, null); + } + + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final String dataBaseName, + final Consumer handleFailure) { + assertDataEventuallyOnEnv( + env, sql, expectedHeader, expectedResSet, 600, dataBaseName, handleFailure); + } + + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final DataNodeWrapper dataNodeWrapper, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final String dataBaseName) { + assertDataEventuallyOnEnv( + env, dataNodeWrapper, sql, expectedHeader, expectedResSet, 600, dataBaseName, null); + } + + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final long timeoutSeconds, + final String databaseName, + final Consumer handleFailure) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; + try (Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + if (databaseName != null) { + statement.execute("use " + databaseName); + } + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] && System.currentTimeMillis() - startTime > timeoutSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } + if (sql != null && !sql.isEmpty()) { + TestUtils.assertResultSetEqual( + executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); + } + } catch (Exception e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + Assert.fail(); + } catch (Error e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + throw e; + } + }); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + // Note that this class will accept any exceptions + public static void assertAlwaysFail(final BaseEnv env, final String sql) { + assertAlwaysFail(env, sql, 10); + } + + public static void assertAlwaysFail( + final BaseEnv env, final String sql, final long consistentSeconds) { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(consistentSeconds, TimeUnit.SECONDS) + .failFast(() -> Assert.assertThrows(Exception.class, () -> statement.executeQuery(sql))); + } catch (final Exception e) { + fail(e.getMessage()); + } + } + + public static void assertDataEventuallyOnEnv( + final BaseEnv env, + final DataNodeWrapper dataNodeWrapper, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final long timeoutSeconds, + final String databaseName, + final Consumer handleFailure) { + try (Connection connection = + env.getConnection( + dataNodeWrapper, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + if (databaseName != null) { + statement.execute("use " + databaseName); + } + if (sql != null && !sql.isEmpty()) { + TestUtils.assertResultSetEqual( + executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); + } + } catch (Exception e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + Assert.fail(); + } catch (Error e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + throw e; + } + }); + } catch (Exception e) { + fail(e.getMessage()); + } + } + public static void assertDataEventuallyOnEnv( BaseEnv env, String sql, Map expectedHeaderWithResult) { assertDataEventuallyOnEnv(env, sql, expectedHeaderWithResult, 600); @@ -730,6 +1517,8 @@ public static void assertDataEventuallyOnEnv( public static void assertDataEventuallyOnEnv( BaseEnv env, String sql, Map expectedHeaderWithResult, long timeoutSeconds) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; try (Connection connection = env.getConnection(); Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures @@ -741,6 +1530,13 @@ public static void assertDataEventuallyOnEnv( .untilAsserted( () -> { try { + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] && System.currentTimeMillis() - startTime > timeoutSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } TestUtils.assertSingleResultSetEqual( executeQueryWithRetry(statement, sql), expectedHeaderWithResult); } catch (Exception e) { @@ -754,8 +1550,29 @@ public static void assertDataEventuallyOnEnv( } public static void assertDataAlwaysOnEnv( - BaseEnv env, String sql, String expectedHeader, Set expectedResSet) { - assertDataAlwaysOnEnv(env, sql, expectedHeader, expectedResSet, 10); + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet) { + assertDataAlwaysOnEnv(env, sql, expectedHeader, expectedResSet, 10, (String) null); + } + + public static void assertDataAlwaysOnEnv( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedResSet, + final String database) { + assertDataAlwaysOnEnv(env, sql, expectedHeader, expectedResSet, 10, database); + } + + public static void assertDataAlwaysOnEnv( + BaseEnv env, + String sql, + String expectedHeader, + Set expectedResSet, + Consumer handleFailure) { + assertDataAlwaysOnEnv(env, sql, expectedHeader, expectedResSet, 10, handleFailure); } public static void assertDataAlwaysOnEnv( @@ -763,7 +1580,103 @@ public static void assertDataAlwaysOnEnv( String sql, String expectedHeader, Set expectedResSet, - long consistentSeconds) { + long consistentSeconds, + final String database) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; + try (Connection connection = + env.getConnection( + Objects.isNull(database) ? TREE_SQL_DIALECT : BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(consistentSeconds, TimeUnit.SECONDS) + .failFast( + () -> { + try { + if (Objects.nonNull(database)) { + statement.execute("use " + database); + } + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] + && System.currentTimeMillis() - startTime > consistentSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } + TestUtils.assertResultSetEqual( + executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); + } catch (Exception e) { + Assert.fail(); + } + }); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + public static void assertDataAlwaysOnEnv( + BaseEnv env, + String sql, + String expectedHeader, + Set expectedResSet, + long consistentSeconds, + Consumer handleFailure) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; + try (Connection connection = env.getConnection(); + Statement statement = connection.createStatement()) { + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(consistentSeconds, TimeUnit.SECONDS) + .failFast( + () -> { + try { + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] + && System.currentTimeMillis() - startTime > consistentSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } + TestUtils.assertResultSetEqual( + executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); + } catch (Exception e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + Assert.fail(); + } catch (Error e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + throw e; + } + }); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + public static void assertDataAlwaysOnEnv( + BaseEnv env, + String sql, + String expectedHeader, + Set expectedResSet, + long consistentSeconds, + String database, + Consumer handleFailure) { + final long startTime = System.currentTimeMillis(); + final boolean[] flushed = {false}; try (Connection connection = env.getConnection(); Statement statement = connection.createStatement()) { // Keep retrying if there are execution failures @@ -775,10 +1688,29 @@ public static void assertDataAlwaysOnEnv( .failFast( () -> { try { + if (database != null) { + statement.execute("use " + database); + } + // For IoTV2 batch mode, the pipe receiver may need to flush because the replica + // sync requires tsFile to process. We flush in the middle of assertion because we + // don't know when the data reaches the receiver in general cases + if (!flushed[0] + && System.currentTimeMillis() - startTime > consistentSeconds >> 1) { + flushed[0] = true; + statement.execute("flush"); + } TestUtils.assertResultSetEqual( executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet); } catch (Exception e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } Assert.fail(); + } catch (Error e) { + if (handleFailure != null) { + handleFailure.accept(e.getMessage()); + } + throw e; } }); } catch (Exception e) { @@ -786,4 +1718,50 @@ public static void assertDataAlwaysOnEnv( fail(); } } + + public static void restartDataNodes() { + EnvFactory.getEnv().shutdownAllDataNodes(); + EnvFactory.getEnv().startAllDataNodes(); + long waitStartMS = System.currentTimeMillis(); + long maxWaitMS = 60_000L; + long retryIntervalMS = 1000; + while (true) { + try (Connection connection = EnvFactory.getEnv().getConnection()) { + break; + } catch (Exception e) { + try { + Thread.sleep(retryIntervalMS); + } catch (InterruptedException ex) { + break; + } + } + long waited = System.currentTimeMillis() - waitStartMS; + if (waited > maxWaitMS) { + fail("Timeout while waiting for datanodes restart"); + } + } + } + + public static void stopForciblyAndRestartDataNodes() { + EnvFactory.getEnv().shutdownForciblyAllDataNodes(); + EnvFactory.getEnv().startAllDataNodes(); + long waitStartMS = System.currentTimeMillis(); + long maxWaitMS = 60_000L; + long retryIntervalMS = 1000; + while (true) { + try (Connection connection = EnvFactory.getEnv().getConnection()) { + break; + } catch (Exception e) { + try { + Thread.sleep(retryIntervalMS); + } catch (InterruptedException ex) { + break; + } + } + long waited = System.currentTimeMillis() - waitStartMS; + if (waited > maxWaitMS) { + fail("Timeout while waiting for datanodes restart"); + } + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java b/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java index 07e0451468836..17e20d91e9ac0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java +++ b/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java @@ -37,33 +37,30 @@ public class IoTDBTestRunner extends BlockJUnit4ClassRunner { private static final Logger logger = IoTDBTestLogger.logger; private IoTDBTestListener listener; - public IoTDBTestRunner(Class testClass) throws InitializationError { + public IoTDBTestRunner(final Class testClass) throws InitializationError { super(testClass); } @Override - public void run(RunNotifier notifier) { - TimeZone.setDefault(TimeZone.getTimeZone("Bejing")); + public void run(final RunNotifier notifier) { + TimeZone.setDefault(TimeZone.getTimeZone("UTC+08:00")); listener = new IoTDBTestListener(this.getName()); notifier.addListener(listener); super.run(notifier); } @Override - protected void runChild(final FrameworkMethod method, RunNotifier notifier) { - Description description = describeChild(method); + protected void runChild(final FrameworkMethod method, final RunNotifier notifier) { + final Description description = describeChild(method); logger.info("Run {}", description.getMethodName()); - long currentTime = System.currentTimeMillis(); + final long currentTime = System.currentTimeMillis(); if (EnvType.getSystemEnvType() != EnvType.MultiCluster) { EnvFactory.getEnv().setTestMethodName(description.getMethodName()); - } else { - // TestMethodName must be set globally in MultiEnvFactory, since the - // cluster environments are not created now - MultiEnvFactory.setTestMethodName(description.getMethodName()); } + MultiEnvFactory.setTestMethodName(description.getMethodName()); super.runChild(method, notifier); - double timeCost = (System.currentTimeMillis() - currentTime) / 1000.0; - String testName = description.getClassName() + "." + description.getMethodName(); + final double timeCost = (System.currentTimeMillis() - currentTime) / 1000.0; + final String testName = description.getClassName() + "." + description.getMethodName(); logger.info("Done {}. Cost: {}s", description.getMethodName(), timeCost); listener.addTestStat(new IoTDBTestStat(testName, timeCost)); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java deleted file mode 100644 index a0a27791a318a..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.itbase.env.BaseEnv; - -import org.junit.After; -import org.junit.Before; - -abstract class AbstractPipeDualAutoIT { - - protected BaseEnv senderEnv; - protected BaseEnv receiverEnv; - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @After - public final void tearDown() { - senderEnv.cleanClusterEnvironment(); - receiverEnv.cleanClusterEnvironment(); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java deleted file mode 100644 index fb541e7dcdb7b..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ /dev/null @@ -1,531 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeAlterIT extends AbstractPipeDualAutoIT { - - @Test - public void testBasicAlterPipe() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - // Create pipe - final String sql = - String.format( - "create pipe a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - long lastCreationTime; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Record last creation time - lastCreationTime = showPipeResult.get(0).creationTime; - } - - // Stop pipe - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("stop pipe a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - } - - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify source ('source.pattern'='root.test2')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (replace) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - "alter pipe a2b replace source ('source'='iotdb-source', 'source.path'='root.test1.**')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); - Assert.assertFalse( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify sink ('sink.batch.enable'='false')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Start pipe - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("start pipe a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Alter pipe (replace) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - "alter pipe a2b replace processor ('processor'='tumbling-time-sampling-processor')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify sink ('connector.batch.enable'='true')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify source ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (replace empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b replace source ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (replace empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b replace processor ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertFalse( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify sink ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertFalse( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - } - - @Test - public void testAlterPipeFailure() { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - // alter non-existed pipe - String sql = - String.format( - "alter pipe a2b modify sink ('node-urls'='%s', 'batch.enable'='true')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(sql); - fail(); - } catch (SQLException ignore) { - } - - // Create pipe - sql = - String.format( - "create pipe a2b with sink ('node-urls'='%s', 'batch.enable'='false')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - } - - @Test - public void testAlterPipeSourceAndProcessor() { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - // Create pipe - final String sql = - String.format( - "create pipe a2b with source ('source' = 'iotdb-source','source.path' = 'root.db.d1.**') with processor ('processor'='tumbling-time-sampling-processor', 'processor.tumbling-time.interval-seconds'='1', 'processor.down-sampling.split-file'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Insert data on sender - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (1000, 1), (1500, 2), (2000, 3), (2500, 4), (3000, 5)", - "flush"))) { - fail(); - } - - // Check data on receiver - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1000,1.0,"); - expectedResSet.add("2000,3.0,"); - expectedResSet.add("3000,5.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.at1,", expectedResSet); - - // Alter pipe (modify 'source.path' and 'processor.tumbling-time.interval-seconds') - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**') modify processor ('processor.tumbling-time.interval-seconds'='2')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Insert data on sender - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d2 (time, at1) values (11000, 1), (11500, 2), (12000, 3), (12500, 4), (13000, 5)", - "flush"))) { - fail(); - } - - // Insert data on sender - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (11000, 1), (11500, 2), (12000, 3), (12500, 4), (13000, 5)", - "flush"))) { - fail(); - } - - // Check data on receiver - expectedResSet.clear(); - expectedResSet.add("11000,null,1.0,"); - expectedResSet.add("13000,null,5.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.db.** where time > 10000", - "Time,root.db.d1.at1,root.db.d2.at1,", - expectedResSet); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java deleted file mode 100644 index c00814cab0d4d..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeDualAutoIT { - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @Test - public void testDoubleLivingAutoConflict() throws Exception { - // Double living is two clusters each with a pipe connecting to the other. - final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String senderIp = senderDataNode.getIp(); - final int senderPort = senderDataNode.getPort(); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - for (int i = 0; i < 100; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); - // Add this property to avoid making self cycle. - extractorAttributes.put("source.forwarding-pipe-requests", "false"); - - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - for (int i = 100; i < 200; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - for (int i = 200; i < 300; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); - // Add this property to avoid to make self cycle. - extractorAttributes.put("source.forwarding-pipe-requests", "false"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - for (int i = 300; i < 400; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - final Set expectedResSet = new HashSet<>(); - for (int i = 0; i < 400; ++i) { - expectedResSet.add(i + ",1.0,"); - } - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - for (int i = 400; i < 500; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - for (int i = 500; i < 600; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - for (int i = 400; i < 600; ++i) { - expectedResSet.add(i + ",1.0,"); - } - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - - @Test - public void testDoubleLivingAutoConflictTemplate() throws Exception { - final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String senderIp = senderDataNode.getIp(); - final int senderPort = senderDataNode.getPort(); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); - extractorAttributes.put("source.forwarding-pipe-requests", "false"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); - extractorAttributes.put("source.forwarding-pipe-requests", "false"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - // Auto extend s1 - "create schema template t1 (s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.db", - "set device template t1 to root.db"))) { - return; - } - - for (int i = 0; i < 200; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - - for (int i = 200; i < 400; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - final Set expectedResSet = new HashSet<>(); - for (int i = 0; i < 400; ++i) { - expectedResSet.add(i + ",1.0,"); - } - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java deleted file mode 100644 index d09db4ec61f6f..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT { - - @Test - public void testAutoDropInHistoricalTransfer() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.mode", "query"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("1,")); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "show pipes", - "ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,", - Collections.emptySet()); - } - } - - @Test - public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, - "insert into root.db.d1(time, s1) values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.mode", "query"); - extractorAttributes.put("extractor.start-time", "1970-01-01T08:00:02+08:00"); - extractorAttributes.put("extractor.end-time", "1970-01-01T08:00:04+08:00"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("3,")); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "show pipes", - "ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,", - Collections.emptySet()); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java deleted file mode 100644 index a017cf112d7aa..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java +++ /dev/null @@ -1,948 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.exception.ClientManagerException; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.cluster.RegionRoleType; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.env.AbstractEnv; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.thrift.TException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT { - - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); - - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(3, 3, 180); - receiverEnv.initClusterEnvironment(3, 3, 180); - } - - @Test - public void testWithAllParametersInLogMode() throws Exception { - testWithAllParameters("log"); - } - - @Test - public void testWithAllParametersInFileMode() throws Exception { - testWithAllParameters("file"); - } - - @Test - public void testWithAllParametersInHybridMode() throws Exception { - testWithAllParameters("hybrid"); - } - - public void testWithAllParameters(final String realtimeMode) throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", - "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", - "flush"))) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("extractor.pattern", "root.db.d1"); - extractorAttributes.put("extractor.history.enable", "true"); - extractorAttributes.put("extractor.history.start-time", "2010-01-01T08:00:00+08:00"); - extractorAttributes.put("extractor.history.end-time", "2010-01-02T08:00:00+08:00"); - extractorAttributes.put("extractor.realtime.enable", "true"); - extractorAttributes.put("extractor.realtime.mode", realtimeMode); - - processorAttributes.put("processor", "do-nothing-processor"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "root"); - connectorAttributes.put("connector.password", "root"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("1,")); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("insert into root.db.d1(time, s1) values (now(), 3)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - } - } - - @Test - public void testPipeAfterDataRegionLeaderStop() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db.d1"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { - return; - } - - final AtomicInteger leaderPort = new AtomicInteger(-1); - final TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq()); - showRegionResp - .getRegionInfoList() - .forEach( - regionInfo -> { - if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { - leaderPort.set(regionInfo.getClientRpcPort()); - } - }); - - int leaderIndex = -1; - for (int i = 0; i < 3; ++i) { - if (senderEnv.getDataNodeWrapper(i).getPort() == leaderPort.get()) { - leaderIndex = i; - try { - senderEnv.shutdownDataNode(i); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - try { - TimeUnit.SECONDS.sleep(1); - } catch (final InterruptedException ignored) { - } - try { - senderEnv.startDataNode(i); - ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - } - } - if (leaderIndex == -1) { // ensure the leader is stopped - fail(); - } - - if (!TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( - senderEnv, - senderEnv.getDataNodeWrapper(leaderIndex), - Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - } - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Create a new pipe and write new data - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db.d2"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d2(time, s1) values (1, 1)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d2", - "count(root.db.d2.s1),", - Collections.singleton("1,")); - } - } - - @Test - public void testPipeAfterRegisterNewDataNode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db.d1"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { - return; - } - - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - final DataNodeWrapper newDataNode = - senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); - if (!TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( - senderEnv, - newDataNode, - Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - } - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // create a new pipe and write new data - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db.d2"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d2(time, s1) values (1, 1)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d2", - "count(root.db.d2.s1),", - Collections.singleton("1,")); - } - } - - @Test - public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final Thread t = - new Thread( - () -> { - for (int i = 0; i < 30; ++i) { - try { - client.createPipe( - new TCreatePipeReq("p" + i, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - } catch (final TException e) { - // Not sure if the "createPipe" has succeeded - e.printStackTrace(); - return; - } - try { - Thread.sleep(100); - } catch (final Exception ignored) { - } - } - }); - t.start(); - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - t.join(); - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(30, showPipeResult.size()); - } - } - - @Test - public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final AtomicInteger succeedNum = new AtomicInteger(0); - final Thread t = - new Thread( - () -> { - try { - for (int i = 0; i < 100; ++i) { - if (TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - succeedNum.incrementAndGet(); - } - Thread.sleep(100); - } - } catch (final InterruptedException ignored) { - } - }); - t.start(); - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - t.join(); - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton(succeedNum.get() + ",")); - - try { - senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); - senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - } catch (final Throwable e) { - e.printStackTrace(); - } - } - } - - @Test - public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - int succeedNum = 0; - for (int i = 0; i < 100; ++i) { - if (TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - succeedNum++; - } - } - - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton(succeedNum + ",")); - - try { - senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); - senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - } catch (final Throwable e) { - e.printStackTrace(); - } - } - } - - @Ignore( - "Currently ignore this test because this test intends to test the behaviour when the sender has" - + " a temporary node joined and then removed, but in reality it just tears it down. In this" - + " circumstance the IT may fail. However, the \"remove\" method is currently not provided thus" - + " we ignore this test now.") - @Test - public void testNewDataNodeFailureParallelToTransferringData() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - int succeedNum = 0; - for (int i = 0; i < 100; ++i) { - if (TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i * 1000))) { - succeedNum++; - } - } - - try { - senderEnv.registerNewDataNode(false); - senderEnv.startDataNode(senderEnv.getDataNodeWrapperList().size() - 1); - senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); - senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton(succeedNum + ",")); - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - } - } - - @Test - public void testSenderRestartWhenTransferring() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - - int succeedNum = 0; - for (int i = 0; i < 100; ++i) { - if (TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i * 1000))) { - succeedNum++; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - try { - TestUtils.restartCluster(senderEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton(succeedNum + ",")); - } - - @Test - public void testConcurrentlyCreatePipeOfSameName() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final AtomicInteger successCount = new AtomicInteger(0); - final List threads = new ArrayList<>(); - for (int i = 0; i < 10; ++i) { - final Thread t = - new Thread( - () -> { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successCount.incrementAndGet(); - } - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (final TException | ClientManagerException | IOException e) { - e.printStackTrace(); - } catch (final Exception e) { - // Fail iff pipe exception occurs - e.printStackTrace(); - fail(e.getMessage()); - } - }); - t.start(); - threads.add(t); - } - - for (Thread t : threads) { - t.join(); - } - Assert.assertEquals(1, successCount.get()); - - successCount.set(0); - for (int i = 0; i < 10; ++i) { - final Thread t = - new Thread( - () -> { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TSStatus status = client.dropPipe("p1"); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successCount.incrementAndGet(); - } - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (final TException | ClientManagerException | IOException e) { - e.printStackTrace(); - } catch (final Exception e) { - // Fail iff pipe exception occurs - e.printStackTrace(); - fail(e.getMessage()); - } - }); - t.start(); - threads.add(t); - } - for (final Thread t : threads) { - t.join(); - } - - // Assert at least 1 drop operation succeeds - Assert.assertTrue(successCount.get() >= 1); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(0, showPipeResult.size()); - } - } - - @Test - public void testCreate10PipesWithSameConnector() throws Exception { - testCreatePipesWithSameConnector(10); - } - - @Test - public void testCreate50PipesWithSameConnector() throws Exception { - testCreatePipesWithSameConnector(50); - } - - @Test - public void testCreate100PipesWithSameConnector() throws Exception { - testCreatePipesWithSameConnector(100); - } - - private void testCreatePipesWithSameConnector(final int pipeCount) throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final AtomicInteger successCount = new AtomicInteger(0); - final List threads = new ArrayList<>(); - for (int i = 0; i < pipeCount; ++i) { - final int finalI = i; - final Thread t = - new Thread( - () -> { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p" + finalI, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - successCount.incrementAndGet(); - } catch (final InterruptedException e) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } catch (final TException | ClientManagerException | IOException e) { - e.printStackTrace(); - } catch (final Exception e) { - // Fail iff pipe exception occurs - e.printStackTrace(); - fail(e.getMessage()); - } - }); - t.start(); - threads.add(t); - } - for (final Thread t : threads) { - t.join(); - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(successCount.get(), showPipeResult.size()); - showPipeResult = - client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; - Assert.assertEquals(successCount.get(), showPipeResult.size()); - } - } - - @Test - public void testNegativeTimestamp() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (0, 1)", - "insert into root.db.d1(time, s1) values (-1, 2)", - "insert into root.db.d1(time, s1) values (1960-01-02T10:00:00+08:00, 2)", - "flush"))) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor", "iotdb-extractor"); - - processorAttributes.put("processor", "do-nothing-processor"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("3,")); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - // Test the correctness of insertRowsNode transmission - "insert into root.db.d1(time, s1) values (-122, 3)", - "insert into root.db.d1(time, s1) values (-123, 3), (now(), 3)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("6,")); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java deleted file mode 100644 index f7c0c63b842e8..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualAutoIT { - - @Override - @Before - public void setUp() { - // Override to enable air-gap - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setPipeAirGapReceiverEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @Test - public void testCompression1() throws Exception { - doTest("iotdb-thrift-connector", "stream", true, "snappy"); - } - - @Test - public void testCompression2() throws Exception { - doTest("iotdb-thrift-connector", "batch", true, "snappy, lzma2"); - } - - @Test - public void testCompression3() throws Exception { - doTest("iotdb-thrift-sync-connector", "stream", false, "snappy, snappy"); - } - - @Test - public void testCompression4() throws Exception { - doTest("iotdb-thrift-sync-connector", "batch", true, "gzip, zstd"); - } - - @Test - public void testCompression5() throws Exception { - doTest("iotdb-air-gap-connector", "stream", false, "lzma2, lz4"); - } - - @Test - public void testCompression6() throws Exception { - doTest("iotdb-air-gap-connector", "batch", true, "lzma2"); - } - - private void doTest( - String connectorType, String realtimeMode, boolean useBatchMode, String compressionTypes) - throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = - connectorType.contains("air-gap") - ? receiverDataNode.getPipeAirGapReceiverPort() - : receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", - "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", - "flush"))) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("extractor.realtime.mode", realtimeMode); - - processorAttributes.put("processor", "do-nothing-processor"); - - connectorAttributes.put("connector", connectorType); - connectorAttributes.put("connector.batch.enable", useBatchMode ? "true" : "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "root"); - connectorAttributes.put("connector.password", "root"); - connectorAttributes.put("connector.compressor", compressionTypes); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (now(), 3)", - "insert into root.db.d1(time, s1) values (now(), 4)", - "insert into root.db.d1(time, s1) values (now(), 5)", - "insert into root.db.d1(time, s1) values (now(), 6)", - "insert into root.db.d1(time, s1) values (now(), 7)", - "insert into root.db.d1(time, s1) values (now(), 8)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("8,")); - } - } - - @Test - public void testZstdCompressorLevel() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (1, 1)", - "insert into root.db.d1(time, s2) values (1, 1)", - "insert into root.db.d1(time, s3) values (1, 1)", - "insert into root.db.d1(time, s4) values (1, 1)", - "insert into root.db.d1(time, s5) values (1, 1)", - "flush"))) { - return; - } - - // Create 5 pipes with different zstd compression levels, p4 and p5 should fail. - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p1" - + " with extractor ('extractor.pattern'='root.db.d1.s1')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='3')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p2" - + " with extractor ('extractor.pattern'='root.db.d1.s2')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='22')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p3" - + " with extractor ('extractor.pattern'='root.db.d1.s3')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='-131072')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p4" - + " with extractor ('extractor.pattern'='root.db.d1.s4')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='-131073')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException e) { - // Make sure the error message in IoTDBConnector.java is returned - Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p5" - + " with extractor ('extractor.pattern'='root.db.d1.s5')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='23')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException e) { - // Make sure the error message in IoTDBConnector.java is returned - Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); - } - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(3, showPipeResult.size()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("3,")); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java deleted file mode 100644 index 87bb4b465e56f..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeConnectorParallelIT extends AbstractPipeDualAutoIT { - @Test - public void testIoTConnectorParallel() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Set expectedResSet = new HashSet<>(); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.parallel.tasks", "3"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.sg1.d1(time, s1) values (0, 1)", - "insert into root.sg1.d1(time, s1) values (1, 2)", - "insert into root.sg1.d1(time, s1) values (2, 3)", - "insert into root.sg1.d1(time, s1) values (3, 4)", - "flush"))) { - return; - } - - expectedResSet.add("0,1.0,"); - expectedResSet.add("1,2.0,"); - expectedResSet.add("2,3.0,"); - expectedResSet.add("3,4.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.sg1.d1.s1,", expectedResSet); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java deleted file mode 100644 index 7fda1250b1bec..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeDualAutoIT { - @Test - public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.realtime.mode", "log"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.realtime-first", "false"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,")))); - } - } - - @Test - public void testSinkTabletFormat() throws Exception { - testSinkFormat("tablet"); - } - - @Test - public void testSinkTsFileFormat() throws Exception { - testSinkFormat("tsfile"); - } - - private void testSinkFormat(final String format) throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.realtime.mode", "forced-log"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.format", format); - connectorAttributes.put("connector.realtime-first", "false"); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)) - .getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)) - .getCode()); - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (4, 1)", - "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet( - new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,")))); - } - } - - @Test - public void testLegacyConnector() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.realtime.mode", "log"); - - connectorAttributes.put("sink", "iotdb-legacy-pipe-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); - - // This version does not matter since it's no longer checked by the legacy receiver - connectorAttributes.put("sink.version", "1.3"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.vehicle.d0.s1,", - Collections.singleton("0,1.0,")); - } - } - - @Test - public void testReceiverAutoCreateByLog() throws Exception { - testReceiverAutoCreate( - new HashMap() { - { - put("source.realtime.mode", "forced-log"); - } - }); - } - - @Test - public void testReceiverAutoCreateByFile() throws Exception { - testReceiverAutoCreate( - new HashMap() { - { - put("source.realtime.mode", "batch"); - } - }); - } - - @Test - public void testReceiverAutoCreateWithPattern() throws Exception { - testReceiverAutoCreate( - new HashMap() { - { - put("source.realtime.mode", "batch"); - put("source.path", "root.ln.wf01.wt0*.*"); - } - }); - } - - private void testReceiverAutoCreate(final Map extractorAttributes) - throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create timeSeries root.ln.wf01.wt01.boolean boolean", - "create timeSeries root.ln.wf01.wt01.int32 int32", - "create timeSeries root.ln.wf01.wt01.int64 int64", - "create timeSeries root.ln.wf01.wt01.float float", - "create timeSeries root.ln.wf01.wt01.double double", - "create timeSeries root.ln.wf01.wt01.time_stamp timestamp", - "create timeSeries root.ln.wf01.wt01.date date", - "create timeSeries root.ln.wf01.wt01.text text", - "create timeSeries root.ln.wf01.wt01.string string", - "create timeSeries root.ln.wf01.wt01.blob blob", - "create aligned timeSeries root.ln.wf01.wt02(int32 int32, boolean boolean)", - "insert into root.ln.wf01.wt01(time, boolean, int32, int64, float, double, time_stamp, date, text, string, blob) values (20000, false, 123, 321, 13.3, 14.4, now(), '2000-12-13', 'abc', 'def', X'f103')", - "insert into root.ln.wf01.wt02(time, int32, boolean) values (20000, 123, false)", - // For pattern parse - "insert into root.ln.wf01.wt11(time, redundant_data) values (20000, -1)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "show timeSeries root.ln.wf01.wt01.*", - "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", - Collections.unmodifiableSet( - new HashSet<>( - Arrays.asList( - "root.ln.wf01.wt01.boolean,null,root.ln,BOOLEAN,RLE,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.int32,null,root.ln,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.int64,null,root.ln,INT64,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.float,null,root.ln,FLOAT,GORILLA,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.double,null,root.ln,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.time_stamp,null,root.ln,TIMESTAMP,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.date,null,root.ln,DATE,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.text,null,root.ln,TEXT,PLAIN,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.string,null,root.ln,STRING,PLAIN,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.blob,null,root.ln,BLOB,PLAIN,LZ4,null,null,null,null,BASE,")))); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "show devices root.ln.wf01.wt02", - "Device,IsAligned,Template,TTL(ms),", - Collections.singleton("root.ln.wf01.wt02,true,null,INF,")); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java deleted file mode 100644 index aea757e63b391..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java +++ /dev/null @@ -1,1051 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeExtractorIT extends AbstractPipeDualAutoIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - // Disable sender compaction for tsfile determination in loose range test - .setEnableSeqSpaceCompaction(false) - .setEnableUnseqSpaceCompaction(false) - .setEnableCrossSpaceCompaction(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @Test - public void testExtractorValidParameter() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - // ---------------------------------------------------------- // - // Scenario 1: when 'extractor.history.enable' is set to true // - // ---------------------------------------------------------- // - - // Scenario 1.1: test when 'extractor.history.start-time' and 'extractor.history.end-time' are - // not set - final String p1_1 = - String.format( - "create pipe p1_1" - + " with extractor (" - + "'extractor.history.enable'='true')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p1_1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 1.2: test when only 'extractor.history.start-time' is set - final String p1_2 = - String.format( - "create pipe p1_2" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p1_2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 1.3: test when only 'extractor.history.end-time' is set - final String p1_3 = - String.format( - "create pipe p1_3" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p1_3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 1.4: test when 'extractor.history.start-time' equals 'extractor.history.end-time' - final String p1_4 = - String.format( - "create pipe p1_4" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2000.01.01T08:00:00'," - + "'extractor.history.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p1_4); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 1.5: test when 'extractor.history.end-time' is future time - final String p1_5 = - String.format( - "create pipe p1_5" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2000.01.01T08:00:00'," - + "'extractor.history.end-time'='2100.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p1_5); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - assertPipeCount(5); - - // ---------------------------------------------------------------------- // - // Scenario 2: when 'source.start-time' or 'source.end-time' is specified // - // ---------------------------------------------------------------------- // - - // Scenario 2.1: test when only 'source.start-time' is set - final String p2_1 = - String.format( - "create pipe p2_1" - + " with source (" - + "'source.start-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p2_1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 2.2: test when only 'source.end-time' is set - final String p2_2 = - String.format( - "create pipe p2_2" - + " with source (" - + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p2_2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 2.3: test when 'source.start-time' equals 'source.end-time' - final String p2_3 = - String.format( - "create pipe p2_3" - + " with source (" - + "'source.start-time'='2000.01.01T08:00:00'," - + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p2_3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Scenario 2.4: test when only when 'source.start-time' is less than 'source.end-time' - final String p2_4 = - String.format( - "create pipe p2_4" - + " with source (" - + "'source.start-time'='2000.01.01T08:00:00'," - + "'source.end-time'='2100.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p2_4); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - assertPipeCount(9); - - // ---------------------------------------------------------------------- // - // Scenario 3: when 'source.start-time' or 'source.end-time' is timestamp // - // ---------------------------------------------------------------------- // - - // Scenario 3.1: test when 'source.start-time' is timestamp string - final String p3_1 = - String.format( - "create pipe p3_1" - + " with source (" - + "'source.start-time'='1000'," - + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p3_1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - assertPipeCount(10); - } - - @Test - public void testExtractorInvalidParameter() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - // Scenario 1: invalid 'extractor.history.start-time' - final String formatString = - String.format( - "create pipe p1" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'=%s)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - "%s", receiverIp, receiverPort); - - final List invalidStartTimes = - Arrays.asList("''", "null", "'null'", "'-1000-01-01T00:00:00'", "'2000-01-01T00:00:0'"); - for (final String invalidStartTime : invalidStartTimes) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(String.format(formatString, invalidStartTime)); - fail(); - } catch (final SQLException ignored) { - } - } - assertPipeCount(0); - - // Scenario 2: can not set 'extractor.history.enable' and 'extractor.realtime.enable' both to - // false - final String p2 = - String.format( - "create pipe p2" - + " with extractor (" - + "'extractor.history.enable'='false'," - + "'extractor.realtime.enable'='false')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p2); - fail(); - } catch (final SQLException ignored) { - } - assertPipeCount(0); - - // Scenario 3: test when 'extractor.history.start-time' is greater than - // 'extractor.history.end-time' - final String p3 = - String.format( - "create pipe p3" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2001.01.01T08:00:00'," - + "'extractor.history.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p3); - fail(); - } catch (final SQLException ignored) { - } - assertPipeCount(0); - - // Scenario 4: test when 'source.start-time' is greater than 'source.end-time' - final String p4 = - String.format( - "create pipe p4" - + " with source (" - + "'source.start-time'='2001.01.01T08:00:00'," - + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p4); - fail(); - } catch (final SQLException ignored) { - } - assertPipeCount(0); - } - - @Test - public void testExtractorPatternMatch() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.nonAligned.1TS (time, s_float) values (now(), 0.5)", - "insert into root.nonAligned.100TS (time, s_float) values (now(), 0.5)", - "insert into root.nonAligned.1000TS (time, s_float) values (now(), 0.5)", - "insert into root.nonAligned.`1(TS)` (time, s_float) values (now(), 0.5)", - "insert into root.nonAligned.6TS.`6` (" - + "time, `s_float(1)`, `s_int(1)`, `s_double(1)`, `s_long(1)`, `s_text(1)`, `s_bool(1)`) " - + "values (now(), 0.5, 1, 1.5, 2, \"text1\", true)", - "insert into root.aligned.1TS (time, s_float) aligned values (now(), 0.5)", - "insert into root.aligned.100TS (time, s_float) aligned values (now(), 0.5)", - "insert into root.aligned.1000TS (time, s_float) aligned values (now(), 0.5)", - "insert into root.aligned.`1(TS)` (time, s_float) aligned values (now(), 0.5)", - "insert into root.aligned.6TS.`6` (" - + "time, `s_float(1)`, `s_int(1)`, `s_double(1)`, `s_long(1)`, `s_text(1)`, `s_bool(1)`) " - + "aligned values (now(), 0.5, 1, 1.5, 2, \"text1\", true)"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", null); - extractorAttributes.put("extractor.inclusion", "data.insert"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final List patterns = - Arrays.asList( - "root.db.nonExistPath", // match nothing - "root.nonAligned.1TS.s_float", - "root.nonAligned.6TS.`6`.`s_text(1)`", - "root.aligned.1TS", - "root.aligned.6TS.`6`.`s_int(1)`", - "root.aligned.`1(TS)`", - "root.nonAligned.`1(TS)`", - "root.aligned.6TS.`6`", // match device root.aligned.6TS.`6` - "root.nonAligned.6TS.`6`", // match device root.nonAligned.6TS.`6` - "root.nonAligned.`1`", // match nothing - "root.nonAligned.1", // match device root.nonAligned.1TS, 100TS and 100TS - "root.aligned.1" // match device root.aligned.1TS, 100TS and 100TS - ); - - final List expectedTimeseriesCount = - Arrays.asList(0, 1, 2, 3, 4, 5, 6, 11, 16, 16, 18, 20); - - for (int i = 0; i < patterns.size(); ++i) { - extractorAttributes.replace("extractor.pattern", patterns.get(i)); - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p" + i, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p" + i).getCode()); - // We add flush here because the pipe may be created on the new IoT leader - // and the old leader's data may come as an unclosed historical tsfile - // and is skipped flush when the pipe starts. In this case, the "waitForTsFileClose()" - // may not return until a flush is executed, namely the data transfer relies - // on a flush operation. - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - assertTimeseriesCountOnReceiver(receiverEnv, expectedTimeseriesCount.get(i)); - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "show devices", - "Device,IsAligned,Template,TTL(ms),", - new HashSet<>( - Arrays.asList( - "root.nonAligned.1TS,false,null,INF,", - "root.nonAligned.100TS,false,null,INF,", - "root.nonAligned.1000TS,false,null,INF,", - "root.nonAligned.`1(TS)`,false,null,INF,", - "root.nonAligned.6TS.`6`,false,null,INF,", - "root.aligned.1TS,true,null,INF,", - "root.aligned.100TS,true,null,INF,", - "root.aligned.1000TS,true,null,INF,", - "root.aligned.`1(TS)`,true,null,INF,", - "root.aligned.6TS.`6`,true,null,INF,"))); - } - } - - @Test - public void testMatchingMultipleDatabases() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db1"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - assertTimeseriesCountOnReceiver(receiverEnv, 0); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db1.d1 (time, at1) values (1, 10)", - "insert into root.db2.d1 (time, at1) values (1, 20)", - "flush"))) { - return; - } - - extractorAttributes.replace("extractor.pattern", "root.db2"); - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - assertTimeseriesCountOnReceiver(receiverEnv, 2); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db1.d1 (time, at1) values (2, 11)", - "insert into root.db2.d1 (time, at1) values (2, 21)", - "flush"))) { - return; - } - - extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases - status = - client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db1.d1.at1),count(root.db2.d1.at1),", - Collections.singleton("2,2,")); - } - } - - @Test - public void testHistoryAndRealtime() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (1, 10)", - "insert into root.db.d2 (time, at1) values (1, 20)", - "insert into root.db.d3 (time, at1) values (1, 30)", - "insert into root.db.d4 (time, at1) values (1, 40)", - "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.pattern", "root.db.d2"); - extractorAttributes.put("extractor.history.enable", "false"); - extractorAttributes.put("extractor.realtime.enable", "true"); - TSStatus status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - extractorAttributes.replace("extractor.pattern", "root.db.d3"); - extractorAttributes.replace("extractor.history.enable", "true"); - extractorAttributes.replace("extractor.realtime.enable", "false"); - status = - client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); - - extractorAttributes.replace("extractor.pattern", "root.db.d4"); - extractorAttributes.replace("extractor.history.enable", "true"); - extractorAttributes.replace("extractor.realtime.enable", "true"); - status = - client.createPipe( - new TCreatePipeReq("p4", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p4").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (2, 11)", - "insert into root.db.d2 (time, at1) values (2, 21)", - "insert into root.db.d3 (time, at1) values (2, 31)", - "insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.** where time <= 1", - "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", - Collections.singleton("1,0,1,")); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.** where time >= 2", - "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", - Collections.singleton("2,1,0,")); - } - } - - @Test - public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" - + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", - "insert into root.db.d2 (time, at1)" - + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", - "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db.d1"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.history.enable", "true"); - // 1970-01-01T08:00:02+08:00 - extractorAttributes.put("extractor.history.start-time", "2000"); - extractorAttributes.put("extractor.history.end-time", "1970-01-01T08:00:04+08:00"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),", - Collections.singleton("3,")); - - extractorAttributes.remove("extractor.pattern"); - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),count(root.db.d2.at1),", - Collections.singleton("3,3,")); - } - } - - @Test - public void testExtractorTimeRangeMatch() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // insert history data - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" - + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", - "insert into root.db.d2 (time, at1)" - + " values (6000, 6), (7000, 7), (8000, 8), (9000, 9), (10000, 10)", - "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - extractorAttributes.put("source.inclusion", "data"); - extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); - extractorAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),", - Collections.singleton("3,")); - - // Insert realtime data that overlapped with time range - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d3 (time, at1)" - + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),count(root.db.d3.at1),", - Collections.singleton("3,3,")); - - // Insert realtime data that does not overlap with time range - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d4 (time, at1)" - + " values (6000, 6), (7000, 7), (8000, 8), (9000, 9), (10000, 10)", - "flush"))) { - return; - } - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),count(root.db.d3.at1),", - Collections.singleton("3,3,")); - } - } - - @Test - public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" - + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", - "insert into root.db.d2 (time, at1)" - + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", - "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.pattern", "root.db.d1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); - // 1970-01-01T08:00:04+08:00 - extractorAttributes.put("source.end-time", "4000"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),", - Collections.singleton("3,")); - - extractorAttributes.remove("source.pattern"); - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.at1),count(root.db.d2.at1),", - Collections.singleton("3,3,")); - } - } - - @Test - public void testHistoryLooseRange() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - // TsFile 1, extracted without parse - "insert into root.db.d1 (time, at1, at2)" + " values (1000, 1, 2), (2000, 3, 4)", - // TsFile 2, not extracted because pattern not overlapped - "insert into root.db1.d1 (time, at1, at2)" + " values (1000, 1, 2), (2000, 3, 4)", - "flush"))) { - return; - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - // TsFile 3, not extracted because time range not overlapped - "insert into root.db.d1 (time, at1, at2)" + " values (3000, 1, 2), (4000, 3, 4)", - "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.path", "root.db.d1.at1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.history.start-time", "1500"); - extractorAttributes.put("source.history.end-time", "2500"); - extractorAttributes.put("source.history.loose-range", "time, path"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.** group by level=0", - "count(root.*.*.*),", - Collections.singleton("4,")); - } - } - - @Test - public void testRealtimeLooseRange() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.path", "root.db.d1.at1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.realtime.loose-range", "time, path"); - extractorAttributes.put("source.start-time", "2000"); - extractorAttributes.put("source.end-time", "10000"); - extractorAttributes.put("source.realtime.mode", "batch"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1, at2)" + " values (1000, 1, 2), (3000, 3, 4)", - "flush"))) { - return; - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db1.d1 (time, at1, at2)" + " values (1000, 1, 2), (3000, 3, 4)", - "flush"))) { - return; - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" + " values (5000, 1), (16000, 3)", - "insert into root.db.d1 (time, at1, at2)" + " values (5001, 1, 2), (6001, 3, 4)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(at1) from root.db.d1 where time >= 2000 and time <= 10000", - new HashMap() { - { - put("count(root.db.d1.at1)", "4"); - } - }); - } - } - - private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton(count + ",")); - } - - private void assertPipeCount(int count) throws Exception { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(count, showPipeResult.size()); - // for (TShowPipeInfo showPipeInfo : showPipeResult) { - // System.out.println(showPipeInfo); - // } - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java deleted file mode 100644 index fab2ef9597a4d..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java +++ /dev/null @@ -1,452 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeIdempotentIT extends AbstractPipeDualAutoIT { - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - // All the schema operations must be under the same database to - // be in the same region, therefore a non-idempotent operation can block the next one - // and fail the IT - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - // Limit the schemaRegion number to 1 to guarantee the after sql executed on the same region - // of the tested idempotent sql. - .setDefaultSchemaRegionGroupNumPerDatabase(1) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @Test - public void testCreateTimeSeriesIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN", - "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("2,")); - } - - @Test - public void testCreateAlignedTimeSeriesIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "CREATE ALIGNED TIMESERIES root.ln.wf01.GPS(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY)", - "create timeSeries root.ln.wf01.wt01.status1(status) with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("3,")); - } - - @Test - public void testCreateTimeSeriesWithAliasIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "create timeSeries root.ln.wf01.wt01.status0(status0) with datatype=BOOLEAN,encoding=PLAIN", - "create timeSeries root.ln.wf01.wt01.status1(status1) with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("2,")); - } - - @Test - public void testInternalCreateTimeSeriesIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "insert into root.ln.wf01.wt01(time, status0) values(now(), false);", - "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("2,")); - } - - @Test - public void testAlterTimeSeriesAddTagIdempotent() throws Exception { - testIdempotent( - Collections.singletonList( - "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN"), - "ALTER timeSeries root.ln.wf01.wt01.status0 ADD TAGS tag3=v3;", - "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("2,")); - } - - @Test - public void testAlterTimeSeriesAddAttrIdempotent() throws Exception { - testIdempotent( - Collections.singletonList( - "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN"), - "ALTER timeSeries root.ln.wf01.wt01.status0 ADD ATTRIBUTES attr1=newV1;", - "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("2,")); - } - - @Test - public void testAlterTimeSeriesRenameIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN", - "ALTER timeSeries root.ln.wf01.wt01.status0 ADD ATTRIBUTES attr1=newV1;"), - "ALTER timeSeries root.ln.wf01.wt01.status0 RENAME attr1 TO newAttr1;", - "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("2,")); - } - - @Test - public void testDeleteTimeSeriesIdempotent() throws Exception { - testIdempotent( - Collections.singletonList( - "create timeSeries root.ln.wf01.wt01.status0(status0) with datatype=BOOLEAN,encoding=PLAIN"), - "delete timeSeries root.ln.wf01.wt01.status0", - "create timeSeries root.ln.wf01.wt01.status2(status2) with datatype=BOOLEAN,encoding=PLAIN", - "count timeSeries", - "count(timeseries),", - Collections.singleton("1,")); - } - - @Test - public void testCreateTemplateIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testExtendTemplateIdempotent() throws Exception { - testIdempotent( - Collections.singletonList( - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)"), - "alter schema template t1 add (rest FLOAT encoding=RLE, FLOAT2 TEXT encoding=PLAIN compression=SNAPPY)", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testDropTemplateIdempotent() throws Exception { - testIdempotent( - Collections.singletonList( - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)"), - "drop schema template t1", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testSetTemplateIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.sg1"), - "set schema template t1 to root.sg1", - "create database root.sg2", - "count databases", - "count,", - Collections.singleton("2,")); - } - - @Test - public void testUnsetTemplateIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.sg1", - "set schema template t1 to root.sg1"), - "unset schema template t1 from root.sg1", - "create database root.sg2", - "count databases", - "count,", - Collections.singleton("2,")); - } - - @Test - public void testActivateTemplateIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.sg1", - "set schema template t1 to root.sg1"), - "create timeSeries using device template on root.sg1.d1", - "create timeSeries using device template on root.sg1.d2", - "count timeSeries", - "count(timeseries),", - Collections.singleton("6,")); - } - - @Test - public void testDeactivateTemplateIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.sg1", - "set schema template t1 to root.sg1", - "create timeSeries using device template on root.sg1.d1", - "create timeSeries using device template on root.sg1.d2"), - "delete timeSeries of schema template t1 from root.sg1.*", - "create timeSeries using device template on root.sg1.d3", - "count timeSeries", - "count(timeseries),", - Collections.singleton("3,")); - } - - @Test - public void testCreateDatabaseIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "create database root.sg1", - "create database root.sg2", - "count databases", - "count,", - Collections.singleton("2,")); - } - - @Test - public void testAlterDatabaseIdempotent() throws Exception { - testIdempotent( - Collections.singletonList("create database root.sg1"), - "ALTER DATABASE root.sg1 WITH SCHEMA_REGION_GROUP_NUM=2, DATA_REGION_GROUP_NUM=3;", - "create database root.sg2", - "count databases", - "count,", - Collections.singleton("2,")); - } - - @Test - public void testDropDatabaseIdempotent() throws Exception { - testIdempotent( - Collections.singletonList("create database root.sg1"), - "drop database root.sg1", - "create database root.sg2", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testCreateUserIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "create user `ln_write_user` 'write_pwd'", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testCreateRoleIdempotent() throws Exception { - testIdempotent( - Collections.emptyList(), - "create role `test`", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testGrantRoleToUserIdempotent() throws Exception { - testIdempotent( - Arrays.asList("create user `ln_write_user` 'write_pwd'", "create role `test`"), - "grant role test to ln_write_user", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testRevokeUserIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create user `ln_write_user` 'write_pwd'", - "GRANT READ_DATA, WRITE_DATA ON root.t1.** TO USER ln_write_user;"), - "REVOKE READ_DATA, WRITE_DATA ON root.t1.** FROM USER ln_write_user;", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testRevokeRoleIdempotent() throws Exception { - testIdempotent( - Arrays.asList("create role `test`", "GRANT READ ON root.** TO ROLE test;"), - "REVOKE READ ON root.** FROM ROLE test;", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testRevokeRoleFromUserIdempotent() throws Exception { - testIdempotent( - Arrays.asList( - "create user `ln_write_user` 'write_pwd'", - "create role `test`", - "grant role test to ln_write_user"), - "revoke role test from ln_write_user", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testDropUserIdempotent() throws Exception { - testIdempotent( - Collections.singletonList("create user `ln_write_user` 'write_pwd'"), - "drop user `ln_write_user`", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - @Test - public void testDropRoleIdempotent() throws Exception { - testIdempotent( - Collections.singletonList("create role `test`"), - "drop role test", - "create database root.sg1", - "count databases", - "count,", - Collections.singleton("1,")); - } - - private void testIdempotent( - final List beforeSqlList, - final String testSql, - final String afterSql, - final String afterSqlQuery, - final String expectedHeader, - final Set expectedResSet) - throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, beforeSqlList)) { - return; - } - - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, testSql)) { - return; - } - - // Create an idempotent conflict, after sql shall be executed on the same region as testSql - if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Arrays.asList(testSql, afterSql))) { - return; - } - - // Assume that the afterSql is executed on receiverEnv - TestUtils.assertDataEventuallyOnEnv(receiverEnv, afterSqlQuery, expectedHeader, expectedResSet); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java deleted file mode 100644 index 87e00151db159..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java +++ /dev/null @@ -1,853 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.createUser; -import static org.apache.iotdb.db.it.utils.TestUtils.executeQueryWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; -import static org.apache.iotdb.db.it.utils.TestUtils.tryExecuteNonQueriesWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.tryExecuteNonQueryWithRetry; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeDualAutoIT { - @Test - public void testLifeCycleWithHistoryEnabled() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - expectedResSet.add("2,2.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - expectedResSet.add("3,3.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testLifeCycleWithHistoryDisabled() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - - extractorAttributes.put("extractor.history.enable", "false"); - // start-time and end-time should not work - extractorAttributes.put("extractor.history.start-time", "0001.01.01T00:00:00"); - extractorAttributes.put("extractor.history.end-time", "2100.01.01T00:00:00"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create database root.ln", - "create timeseries root.db.d1.s2 with datatype=BOOLEAN,encoding=PLAIN", - "insert into root.db.d1(time, s1) values (2, 2)", - "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select s1 from root.db.d1", - "Time,root.db.d1.s1,", - Collections.singleton("2,2.0,")); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("2,")); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count databases", "count,", Collections.singleton("2,")); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create database root.ln0", - "create timeseries root.db.d1.s3 with datatype=BOOLEAN,encoding=PLAIN", - "insert into root.db.d1(time, s1) values (3, 3)", - "flush"))) { - return; - } - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "select s1 from root.db.d1", - "Time,root.db.d1.s1,", - Collections.singleton("2,2.0,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("2,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "count databases", "count,", Collections.singleton("2,")); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select s1 from root.db.d1", - "Time,root.db.d1.s1,", - new HashSet<>(Arrays.asList("2,2.0,", "3,3.0,"))); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("3,")); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count databases", "count,", Collections.singleton("3,")); - } - } - - @Test - public void testLifeCycleLogMode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.mode", "log"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - expectedResSet.add("2,2.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testLifeCycleFileMode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.mode", "file"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - expectedResSet.add("2,2.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testLifeCycleHybridMode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.mode", "hybrid"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - expectedResSet.add("2,2.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testLifeCycleWithClusterRestart() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Set expectedResSet = new HashSet<>(); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - expectedResSet.add("1,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - expectedResSet.add("2,2.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - try (final SyncConfigNodeIServiceClient ignored = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - expectedResSet.add("3,3.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testReceiverRestartWhenTransferring() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final AtomicInteger succeedNum = new AtomicInteger(0); - final Thread t = - new Thread( - () -> { - try { - for (int i = 0; i < 100; ++i) { - if (TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - succeedNum.incrementAndGet(); - } - Thread.sleep(100); - } - } catch (InterruptedException ignored) { - } - }); - t.start(); - - try { - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - try { - t.interrupt(); - t.join(); - } catch (Throwable ignored) { - } - return; - } - - t.join(); - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton(succeedNum.get() + ",")); - } - } - - @Test - public void testReceiverAlreadyHaveTimeSeries() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - Thread.sleep(5000); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - } - } - - @Test - public void testDoubleLiving() throws Exception { - // Double living is two clusters with pipes connecting each other. - final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String senderIp = senderDataNode.getIp(); - final int senderPort = senderDataNode.getPort(); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - for (int i = 0; i < 100; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - // Add this property to avoid to make self cycle. - connectorAttributes.put("source.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - for (int i = 100; i < 200; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - - for (int i = 200; i < 300; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - // Add this property to avoid to make self cycle. - connectorAttributes.put("source.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - for (int i = 300; i < 400; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - final Set expectedResSet = new HashSet<>(); - for (int i = 0; i < 400; ++i) { - expectedResSet.add(i + ",1.0,"); - } - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - for (int i = 400; i < 500; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - return; - } - for (int i = 500; i < 600; ++i) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { - return; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { - return; - } - - for (int i = 400; i < 600; ++i) { - expectedResSet.add(i + ",1.0,"); - } - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - } - - @Test - public void testPermission() { - createUser(senderEnv, "test", "test123"); - - assertNonQueryTestFail( - senderEnv, - "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='iotdb-thrift-connector',\n" - + " 'connector.ip'='127.0.0.1',\n" - + " 'connector.port'='6668'\n" - + ")", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - assertNonQueryTestFail( - senderEnv, - "drop pipe testPipe", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - assertTestFail( - senderEnv, - "show pipes", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - assertNonQueryTestFail( - senderEnv, - "start pipe testPipe", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - assertNonQueryTestFail( - senderEnv, - "stop pipe testPipe", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - - assertNonQueryTestFail( - senderEnv, - "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - assertNonQueryTestFail( - senderEnv, - "drop pipePlugin TestProcessor", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - assertTestFail( - senderEnv, - "show pipe plugins", - "803: No permissions for this operation, please add privilege USE_PIPE", - "test", - "test123"); - - grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.USE_PIPE); - - tryExecuteNonQueryWithRetry( - senderEnv, - "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='iotdb-thrift-connector',\n" - + " 'connector.ip'='127.0.0.1',\n" - + " 'connector.port'='6668'\n" - + ")", - "test", - "test123"); - executeQueryWithRetry(senderEnv, "show pipes", "test", "test123"); - tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), - "test", - "test123"); - - assertNonQueryTestFail( - senderEnv, - "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", - "1603: The scheme of URI is not set, please specify the scheme of URI.", - "test", - "test123"); - tryExecuteNonQueryWithRetry(senderEnv, "drop pipePlugin TestProcessor", "test", "test123"); - executeQueryWithRetry(senderEnv, "show pipe plugins", "test", "test123"); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java deleted file mode 100644 index 4d86fd5ed55b9..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.isession.ISession; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeDualAutoIT { - - // Test dimensions: - // 1. is or not aligned - // 2. is or not parsed - // 3. session insertRecord, session insertTablet, SQL insert - // 4. partial null, all null - // 5. one row or more (TODO) - // 6. more data types (TODO) - - private enum InsertType { - SESSION_INSERT_RECORD, - SESSION_INSERT_TABLET, - SQL_INSERT, - } - - private static final Map> INSERT_NULL_VALUE_MAP = new HashMap<>(); - - private static final List CREATE_TIMESERIES_SQL = - Arrays.asList( - "create timeseries root.sg.d1.s0 with datatype=float", - "create timeseries root.sg.d1.s1 with datatype=float"); - - private static final List CREATE_ALIGNED_TIMESERIES_SQL = - Collections.singletonList("create aligned timeseries root.sg.d1(s0 float, s1 float)"); - - private final String deviceId = "root.sg.d1"; - private final List measurements = Arrays.asList("s0", "s1"); - private final List types = Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT); - - private final List partialNullValues = Arrays.asList(null, 25.34F); - private final List allNullValues = Arrays.asList(null, null); - - private Tablet partialNullTablet; - private Tablet allNullTablet; - - private void constructTablet() { - final MeasurementSchema[] schemas = new MeasurementSchema[2]; - for (int i = 0; i < schemas.length; i++) { - schemas[i] = new MeasurementSchema(measurements.get(i), types.get(i)); - } - - final BitMap[] bitMapsForPartialNull = new BitMap[2]; - bitMapsForPartialNull[0] = new BitMap(1); - bitMapsForPartialNull[0].markAll(); - bitMapsForPartialNull[1] = new BitMap(1); - - final BitMap[] bitMapsForAllNull = new BitMap[2]; - bitMapsForAllNull[0] = new BitMap(1); - bitMapsForAllNull[0].markAll(); - bitMapsForAllNull[1] = new BitMap(1); - bitMapsForAllNull[1].markAll(); - - final Object[] valuesForPartialNull = new Object[2]; - valuesForPartialNull[0] = new float[] {0F}; - valuesForPartialNull[1] = new float[] {25.34F}; - - final Object[] valuesForAllNull = new Object[2]; - valuesForAllNull[0] = new float[] {0F}; - valuesForAllNull[1] = new float[] {0F}; - - partialNullTablet = new Tablet(deviceId, Arrays.asList(schemas), 1); - partialNullTablet.values = valuesForPartialNull; - partialNullTablet.timestamps = new long[] {3}; - partialNullTablet.rowSize = 1; - partialNullTablet.bitMaps = bitMapsForPartialNull; - - allNullTablet = new Tablet(deviceId, Arrays.asList(schemas), 1); - allNullTablet.values = valuesForAllNull; - allNullTablet.timestamps = new long[] {4}; - allNullTablet.rowSize = 1; - allNullTablet.bitMaps = bitMapsForAllNull; - } - - @Override - @Before - public void setUp() { - super.setUp(); - - constructTablet(); - - // Initialize INSERT_NULL_VALUE_MAP - INSERT_NULL_VALUE_MAP.put( - InsertType.SESSION_INSERT_RECORD, - (isAligned) -> { - try { - try (final ISession session = senderEnv.getSessionConnection()) { - if (isAligned) { - session.insertAlignedRecord(deviceId, 3, measurements, types, partialNullValues); - session.insertAlignedRecord(deviceId, 4, measurements, types, allNullValues); - } else { - session.insertRecord(deviceId, 3, measurements, types, partialNullValues); - session.insertRecord(deviceId, 4, measurements, types, allNullValues); - } - } catch (StatementExecutionException e) { - fail(e.getMessage()); - } - } catch (IoTDBConnectionException e) { - fail(e.getMessage()); - } - }); - - INSERT_NULL_VALUE_MAP.put( - InsertType.SESSION_INSERT_TABLET, - (isAligned) -> { - try { - try (final ISession session = senderEnv.getSessionConnection()) { - if (isAligned) { - session.insertAlignedTablet(partialNullTablet); - session.insertAlignedTablet(allNullTablet); - } else { - session.insertTablet(partialNullTablet); - session.insertTablet(allNullTablet); - } - } catch (StatementExecutionException e) { - fail(e.getMessage()); - } - } catch (IoTDBConnectionException e) { - fail(e.getMessage()); - } - }); - - INSERT_NULL_VALUE_MAP.put( - InsertType.SQL_INSERT, - (isAligned) -> { - // Partial null - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - isAligned - ? Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) aligned values (3, null, 25.34)") - : Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) values (3, null, 25.34)"))) { - fail(); - } - // All null - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - isAligned - ? Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) aligned values (4, null, null)") - : Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) values (4, null, null)"))) { - fail(); - } - }); - } - - private void testInsertNullValueTemplate( - final InsertType insertType, final boolean isAligned, final boolean withParsing) - throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - if (withParsing) { - extractorAttributes.put("start-time", "1970-01-01T08:00:00.000+08:00"); - extractorAttributes.put("end-time", "1970-01-01T09:00:00.000+08:00"); - extractorAttributes.put("extractor.pattern", "root.sg.d1"); - } - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("test", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - receiverEnv, isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL)) { - fail(); - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL)) { - fail(); - } - - INSERT_NULL_VALUE_MAP.get(insertType).accept(isAligned); - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { - fail(); - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.sg.d1.s0),count(root.sg.d1.s1),", - Collections.singleton("0,1,")); - } - - // ---------------------- // - // Scenario 1: SQL Insert // - // ---------------------- // - @Test - public void testSQLInsertWithParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, false, true); - } - - @Test - public void testSQLInsertWithoutParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, false, false); - } - - @Test - public void testSQLInsertAlignedWithParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, true, true); - } - - @Test - public void testSQLInsertAlignedWithoutParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, true, false); - } - - // --------------------------------- // - // Scenario 2: Session Insert Record // - // --------------------------------- // - @Test - public void testSessionInsertRecordWithParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, false, true); - } - - @Test - public void testSessionInsertRecordWithoutParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, false, false); - } - - @Test - public void testSessionInsertRecordAlignedWithParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, true, true); - } - - @Test - public void testSessionInsertRecordAlignedWithoutParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, true, false); - } - - // --------------------------------- // - // Scenario 3: Session Insert Tablet // - // --------------------------------- // - @Test - public void testSessionInsertTabletWithParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, true); - } - - @Test - public void testSessionInsertTabletWithoutParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, false); - } - - @Test - public void testSessionInsertTabletAlignedWithParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, true); - } - - @Test - public void testSessionInsertTabletAlignedWithoutParsing() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, false); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipePatternFormatIT.java deleted file mode 100644 index 1d80e175344a1..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipePatternFormatIT.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipePatternFormatIT extends AbstractPipeDualAutoIT { - @Test - public void testPrefixPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (1, 1)", - "insert into root.db2.d1(time, s) values (1, 1)"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.db.d1.s"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.**", "Time,root.db.d1.s,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testIotdbPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (1, 1)", - "insert into root.db2.d1(time, s) values (1, 1)"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.path", "root.**.d1.s*"); - // When path is set, pattern should be ignored - extractorAttributes.put("extractor.pattern", "root"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,1.0,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", - expectedResSet); - } - } - - @Test - public void testIotdbPatternWithLegacySyntax() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (1, 1)", - "insert into root.db2.d1(time, s) values (1, 1)"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.pattern", "root.**.d1.s*"); - extractorAttributes.put("extractor.pattern.format", "iotdb"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,1.0,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", - expectedResSet); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java deleted file mode 100644 index 373e9d4dbea7e..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java +++ /dev/null @@ -1,492 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.pipe.plugin.builtin.BuiltinPipePlugin; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT { - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - } - - private void innerSetUp( - final String configNodeConsensus, - final String schemaRegionConsensus, - final String dataRegionConsensus, - final int configNodesNum, - final int dataNodesNum, - int schemaRegionReplicationFactor, - int dataRegionReplicationFactor) { - schemaRegionReplicationFactor = Math.min(schemaRegionReplicationFactor, dataNodesNum); - dataRegionReplicationFactor = Math.min(dataRegionReplicationFactor, dataNodesNum); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(dataRegionConsensus) - .setSchemaReplicationFactor(schemaRegionReplicationFactor) - .setDataReplicationFactor(dataRegionReplicationFactor); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(configNodeConsensus) - .setSchemaRegionConsensusProtocolClass(schemaRegionConsensus) - .setDataRegionConsensusProtocolClass(dataRegionConsensus) - .setSchemaReplicationFactor(schemaRegionReplicationFactor) - .setDataReplicationFactor(dataRegionReplicationFactor); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); - receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); - } - - @Test - public void test1C1DWithRatisRatisIot() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 1, - 1, - 1, - 1); - doTest(); - } - - @Test - public void test1C1DWithSimpleSimpleIot() throws Exception { - innerSetUp( - ConsensusFactory.SIMPLE_CONSENSUS, - ConsensusFactory.SIMPLE_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 1, - 1, - 1, - 1); - doTest(); - } - - @Test - public void test1C1DWithRatisRatisSimple() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.SIMPLE_CONSENSUS, - 1, - 1, - 1, - 1); - doTest(); - } - - @Test - public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 3, - 3, - 3, - 3); - doTest(); - } - - @Test - public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 3, - 3, - 3, - 2); - doTest(); - } - - @Test - public void testPipeOnBothSenderAndReceiver() throws Exception { - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(3) - .setDataReplicationFactor(2); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(1) - .setDataReplicationFactor(1); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(3, 3); - receiverEnv.initClusterEnvironment(1, 1); - - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("1,")); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - } - - final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); - final String senderIp = senderDataNode.getIp(); - final int senderPort = senderDataNode.getPort(); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, "insert into root.db.d1(time, s1) values (2, 2)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - } - } - - private void doTest() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("1,")); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - Thread.sleep(5000); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - } - } - - @Test - public void testSyncConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); - } - - @Test - public void testAsyncConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); - } - - @Test - public void testAirGapConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); - } - - private void doTestUseNodeUrls(String connectorName) throws Exception { - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setPipeAirGapReceiverEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(1) - .setDataReplicationFactor(1) - .setEnableSeqSpaceCompaction(false) - .setEnableUnseqSpaceCompaction(false) - .setEnableCrossSpaceCompaction(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setPipeAirGapReceiverEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(3) - .setDataReplicationFactor(2); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 3); - - final StringBuilder nodeUrlsBuilder = new StringBuilder(); - for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { - if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { - // Use default port for convenience - nodeUrlsBuilder - .append(wrapper.getIp()) - .append(":") - .append(wrapper.getPipeAirGapReceiverPort()) - .append(","); - } else { - nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); - } - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - // Test mods transfer - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (1, 1)", - "insert into root.db.d1(time, s1) values (3, 1)", - "flush", - "delete from root.db.d1.s1 where time > 2"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", connectorName); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); - - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.mods.enable", "true"); - - // Test forced-log mode, in open releases this might be "file" - extractorAttributes.put("source.realtime.mode", "forced-log"); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("1,")); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(s1) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton("2,")); - - // Test metadata - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create timeseries root.db.d1.s2 with datatype=BOOLEAN,encoding=PLAIN", - "create database root.test1"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("2,")); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count databases", "count,", Collections.singleton("2,")); - - // Test file mode - extractorAttributes.put("source.inclusion", "data"); - extractorAttributes.replace("source.realtime.mode", "file"); - - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(s1) from root.db.d1", - "count(root.db.d1.s1),", - Collections.singleton("3,")); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java deleted file mode 100644 index 7463b6ab06d82..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualAutoIT { - @Test - public void testPipeSwitchStatus() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - status = - client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p2") && o.state.equals("RUNNING"))); - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p3") && o.state.equals("RUNNING"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p3").getCode()); - - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p2") && o.state.equals("RUNNING"))); - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); - - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); - - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); - } - } - - @Test - public void testPipeIllegallySwitchStatus() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - - status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), status.getCode()); - - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); - Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); - Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); - } - } - - @Test - public void testDropPipeAndCreateAgain() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "drop database root.**")) { - return; - } - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(0, showPipeResult.size()); - - status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); - } - } - - @Test - public void testWrongPipeName() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); - Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); - - Assert.assertEquals( - TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("").getCode()); - Assert.assertEquals( - TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("p0").getCode()); - Assert.assertEquals( - TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("p").getCode()); - Assert.assertEquals( - TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("*").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertTrue( - showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java deleted file mode 100644 index b0293361b6255..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java +++ /dev/null @@ -1,721 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.autocreate; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeDualAutoIT { - @Test - public void testValidPipeName() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - final List validPipeNames = - Arrays.asList("Pipe_1", "null", "`33`", "`root`", "中文", "with"); - final List expectedPipeNames = - Arrays.asList("Pipe_1", "null", "33", "root", "中文", "with"); - for (final String pipeName : validPipeNames) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe %s" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - pipeName, receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - for (final String pipeName : expectedPipeNames) { - Assert.assertTrue( - showPipeResult.stream() - .anyMatch((o) -> o.id.equals(pipeName) && o.state.equals("RUNNING"))); - } - - for (final String pipeName : validPipeNames) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(String.format("drop pipe %s", pipeName)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(0, showPipeResult.size()); - } - } - - @Test - public void testRevertParameterOrder() { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p1" - + " with extractor (" - + "'extractor.realtime.mode'='hybrid'," - + "'extractor.history.enable'='false') " - + " with connector (" - + "'connector.batch.enable'='false', " - + "'connector.port'='%s'," - + "'connector.ip'='%s'," - + "'connector'='iotdb-thrift-connector')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignore) { - } - } - - @Test - public void testRevertStageOrder() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p1" - + " with connector (" - + "'connector.batch.enable'='false', " - + "'connector.port'='%s'," - + "'connector.ip'='%s'," - + "'connector'='iotdb-thrift-connector') " - + " with extractor (" - + "'extractor.realtime.mode'='hybrid'," - + "'extractor.history.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(0, showPipeResult.size()); - } - } - - @Test - public void testMissingStage() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("create pipe p1"); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("create pipe p2 with extractor ('extractor'='iotdb-extractor')"); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - "create pipe p3" - + " with extractor ('extractor'='iotdb-extractor')" - + " with processor ('processor'='do-nothing-processor')"); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p4" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p5" - + " with extractor ('extractor'='iotdb-extractor')" - + " with processor ('processor'='do-nothing-processor')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(2, showPipeResult.size()); - } - } - - @Test - public void testInvalidParameter() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p1" - + " with extractor ()" - + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p2" - + " with extractor ('extractor'='invalid-param')" - + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p3" - + " with extractor ()" - + " with processor ('processor'='invalid-param')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p4" - + " with extractor ()" - + " with processor ()" - + " with connector (" - + "'connector'='invalid-param'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - } - } - - @Test - public void testBrackets() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe extractor1" - + " with extractor ('extractor'='iotdb-extractor')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe extractor2" - + " with extractor (\"extractor\"=\"iotdb-extractor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe extractor3" - + " with extractor ('extractor'=\"iotdb-extractor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe extractor4" - + " with extractor (extractor=iotdb-extractor)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe extractor5" - + " with extractor ('extractor'=`iotdb-extractor`)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe processor1" - + " with processor ('processor'='do-nothing-processor')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe processor2" - + " with processor (\"processor\"=\"do-nothing-processor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe processor3" - + " with processor ('processor'=\"do-nothing-processor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe processor4" - + " with processor (processor=do-nothing-processor)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe processor5" - + " with processor ('processor'=`do-nothing-processor`)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe connector1" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe connector2" - + " with connector (" - + "\"connector\"=\"iotdb-thrift-connector\"," - + "\"connector.ip\"=\"%s\"," - + "\"connector.port\"=\"%s\"," - + "\"connector.batch.enable\"=\"false\")", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe connector3" - + " with connector (" - + "'connector'=\"iotdb-thrift-connector\"," - + "\"connector.ip\"='%s'," - + "'connector.port'=\"%s\"," - + "\"connector.batch.enable\"='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe connector4" - + " with connector (" - + "connector=iotdb-thrift-connector," - + "connector.ip=%s," - + "connector.port=%s," - + "connector.batch.enable=false)", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe connector5" - + " with connector (" - + "'connector'=`iotdb-thrift-connector`," - + "'connector.ip'=`%s`," - + "'connector.port'=`%s`," - + "'connector.batch.enable'=`false`)", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(9, showPipeResult.size()); - } - } - - @Test - public void testShowPipeWithWrongPipeName() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - connectorAttributes.replace("connector.batch.enable", "true"); - - status = - client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(3, showPipeResult.size()); - - showPipeResult = client.showPipe(new TShowPipeReq().setPipeName("p1")).pipeInfoList; - Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); - - // Show all pipes whose connector is also used by p1. - // p1 and p2 share the same connector parameters, so they have the same connector. - showPipeResult = - client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; - Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); - Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); - Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); - } - } - - @Test - public void testInclusionPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Empty inclusion - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p2" - + " with extractor ('extractor.inclusion'='schema, auth.role', 'extractor.inclusion.exclusion'='all')" - + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - // Invalid inclusion - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p3" - + " with extractor ('extractor.inclusion'='wrong')" - + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - // Invalid exclusion - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p4" - + " with extractor ('extractor.inclusion.exclusion'='wrong')" - + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException ignored) { - } - - // Valid - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p4" - + " with extractor ('extractor.inclusion'='all', 'extractor.inclusion.exclusion'='schema.database.drop, auth.role')" - + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - Assert.assertEquals(1, showPipeResult.size()); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java new file mode 100644 index 0000000000000..635dbcd97bd17 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java @@ -0,0 +1,822 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.RpcUtils; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +/** + * This class is specifically designed for table model testing and is used for data insertion. + * Please note that the Table pattern defined by this class is fixed, and it is recommended to use + * all methods together to ensure data consistency and integrity. + * + *

This class provides a structured approach to inserting data, suitable for scenarios where + * batch data insertion is required. Due to the fixed pattern, users should ensure that the data + * format matches the pattern defined by the class. + */ +public class TableModelUtils { + + public static void createDataBaseAndTable( + final BaseEnv baseEnv, final String table, final String database) { + try (Connection connection = baseEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists " + database); + statement.execute("use " + database); + statement.execute( + String.format( + "CREATE TABLE %s(s0 string tag, s1 string tag, s2 string tag, s3 string tag,s4 int64 field, s5 float field, s6 string field, s7 timestamp field, s8 int32 field, s9 double field, s10 date field, s11 text field )", + table)); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + public static void createDatabase(final BaseEnv baseEnv, final String database) { + createDatabase(baseEnv, database, Long.MAX_VALUE); + } + + public static void createDatabase(final BaseEnv baseEnv, final String database, final long ttl) { + try (final Connection connection = baseEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create database if not exists " + + database + + (ttl < Long.MAX_VALUE ? " with (ttl=" + ttl + ")" : "")); + } catch (final Exception e) { + fail(e.getMessage()); + } + } + + public static boolean insertData( + final String dataBaseName, + final String tableName, + final int startInclusive, + final int endExclusive, + final BaseEnv baseEnv) { + List list = new ArrayList<>(endExclusive - startInclusive + 1); + for (int i = startInclusive; i < endExclusive; ++i) { + list.add( + String.format( + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); + } + list.add("flush"); + return TestUtils.tryExecuteNonQueriesWithRetry( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list); + } + + public static boolean insertData( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv, + final boolean allowNullValue) { + List list = new ArrayList<>(end - start + 1); + Object[] values = new Object[12]; + Random random = new Random(); + // s0 string, s1 string, s2 string, s3 string, s4 int64, s5 float, s6 string s7 timestamp, s8 + // int32, s9 double, s10 date, s11 text + for (int i = start; i < end; ++i) { + Arrays.fill(values, i); + values[0] = String.format("'t%s'", i); + values[1] = String.format("'t%s'", i); + values[2] = String.format("'t%s'", i); + values[3] = String.format("'t%s'", i); + values[4] = String.format("%s", i); + values[5] = String.format("%s.0", i); + values[6] = String.format("%s", i); + values[7] = String.format("%s", i); + values[8] = String.format("%s", i); + values[9] = String.format("%s.0", i); + values[10] = String.format("'%s'", getDateStr(i)); + values[11] = String.format("'%s'", i); + if (allowNullValue) { + values[random.nextInt(9)] = "null"; + } + list.add( + String.format( + "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8,s9, s10, s11, time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + tableName, + values[0], + values[1], + values[2], + values[3], + values[4], + values[5], + values[6], + values[7], + values[8], + values[9], + values[10], + values[11], + i)); + } + return TestUtils.tryExecuteNonQueriesWithRetry( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list); + } + + public static boolean insertDataNotThrowError( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv) { + List list = new ArrayList<>(end - start + 1); + for (int i = start; i < end; ++i) { + list.add( + String.format( + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); + } + return TestUtils.tryExecuteNonQueriesWithRetry( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list); + } + + public static boolean insertData( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv, + final DataNodeWrapper wrapper) { + List list = new ArrayList<>(end - start + 1); + for (int i = start; i < end; ++i) { + list.add( + String.format( + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); + } + list.add("flush"); + return TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( + baseEnv, wrapper, list, dataBaseName, BaseEnv.TABLE_SQL_DIALECT); + } + + public static boolean insertTablet( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv, + final boolean allowNullValue) { + final Tablet tablet = generateTablet(tableName, start, end, allowNullValue, true); + ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(1); + try (final ITableSession session = tableSessionPool.getSession()) { + session.executeNonQueryStatement("use " + dataBaseName); + session.insert(tablet); + session.executeNonQueryStatement("flush"); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static boolean insertTablet( + final String dataBaseName, final Tablet tablet, final BaseEnv baseEnv) { + try (ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(20); + final ITableSession session = tableSessionPool.getSession()) { + session.executeNonQueryStatement("use " + dataBaseName); + session.insert(tablet); + session.executeNonQueryStatement("flush"); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static void deleteData( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv) { + List list = new ArrayList<>(end - start + 1); + list.add( + String.format("delete from %s where time >= %s and time <= %s", tableName, start, end)); + if (!TestUtils.tryExecuteNonQueriesWithRetry( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list)) { + fail(); + } + } + + // s0 string, s1 string, s2 string, s3 string, s4 int64, s5 float, s6 string s7 timestamp, s8 + // int32, s9 double, s10 date, s11 text + public static Set generateExpectedResults(final int start, final int end) { + Set expectedResSet = new HashSet<>(); + for (int i = start; i < end; ++i) { + final String time = RpcUtils.formatDatetime("default", "ms", i, ZoneOffset.UTC); + expectedResSet.add( + String.format( + "t%s,t%s,t%s,t%s,%s,%s.0,%s,%s,%d,%d.0,%s,%s,%s,", + i, i, i, i, i, i, i, time, i, i, getDateStr(i), i, time)); + } + return expectedResSet; + } + + public static Set generateExpectedResults(final Tablet tablet) { + Set expectedResSet = new HashSet<>(); + List schemas = tablet.getSchemas(); + for (int i = 0; i < tablet.getRowSize(); i++) { + StringBuilder stringBuffer = new StringBuilder(); + for (int j = 0; j < tablet.getSchemas().size(); j++) { + BitMap bitMap = tablet.getBitMaps()[j]; + if (bitMap.isMarked(i)) { + stringBuffer.append("null,"); + continue; + } + switch (schemas.get(j).getType()) { + case TIMESTAMP: + final String time = + RpcUtils.formatDatetime( + "default", "ms", (long) tablet.getValue(i, j), ZoneOffset.UTC); + stringBuffer.append(time); + stringBuffer.append(","); + break; + case DATE: + stringBuffer.append(tablet.getValue(i, j).toString()); + stringBuffer.append(","); + break; + case BLOB: + stringBuffer.append( + BytesUtils.parseBlobByteArrayToString( + ((Binary) tablet.getValue(i, j)).getValues())); + stringBuffer.append(","); + break; + case TEXT: + case STRING: + stringBuffer.append( + new String(((Binary) tablet.getValue(i, j)).getValues(), StandardCharsets.UTF_8)); + stringBuffer.append(","); + break; + case DOUBLE: + stringBuffer.append((double) tablet.getValue(i, j)); + stringBuffer.append(","); + break; + case FLOAT: + stringBuffer.append((float) tablet.getValue(i, j)); + stringBuffer.append(","); + break; + case INT32: + stringBuffer.append((int) tablet.getValue(i, j)); + stringBuffer.append(","); + break; + case INT64: + stringBuffer.append((long) tablet.getValue(i, j)); + stringBuffer.append(","); + break; + } + } + String time = + RpcUtils.formatDatetime("default", "ms", tablet.getTimestamp(i), ZoneOffset.UTC); + stringBuffer.append(time); + stringBuffer.append(","); + expectedResSet.add(stringBuffer.toString()); + } + + return expectedResSet; + } + + public static String generateHeaderResults() { + return "s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time,"; + } + + public static String getQuerySql(final String table) { + return "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from " + table; + } + + public static String getQueryCountSql(final String table) { + return "select count(*) from " + table; + } + + public static void assertData( + final String database, + final String table, + final int start, + final int end, + final BaseEnv baseEnv) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + TableModelUtils.getQuerySql(table), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(start, end), + database); + } + + public static void assertData( + final String database, + final String table, + final Set expectedResults, + final BaseEnv baseEnv, + final Consumer handleFailure) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + TableModelUtils.getQuerySql(table), + TableModelUtils.generateHeaderResults(), + expectedResults, + database, + handleFailure); + } + + public static void assertData( + final String database, + final String table, + final int start, + final int end, + final BaseEnv baseEnv, + final Consumer handleFailure) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + TableModelUtils.getQuerySql(table), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(start, end), + database, + handleFailure); + } + + public static void assertData( + final String database, final String table, final Tablet tablet, final BaseEnv baseEnv) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + TableModelUtils.getQuerySql(table), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(tablet), + database); + } + + public static boolean hasDataBase(final String database, final BaseEnv baseEnv) { + TestUtils.assertDataEventuallyOnEnv(baseEnv, "", "", Collections.emptySet(), database); + return true; + } + + public static void assertCountDataAlwaysOnEnv( + final String database, final String table, final int count, final BaseEnv baseEnv) { + TestUtils.assertDataAlwaysOnEnv( + baseEnv, getQueryCountSql(table), "_col0,", Collections.singleton(count + ","), database); + } + + public static void assertCountData( + final String database, final String table, final int count, final BaseEnv baseEnv) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, getQueryCountSql(table), "_col0,", Collections.singleton(count + ","), database); + } + + public static void assertCountData( + final String database, + final String table, + final int count, + final BaseEnv baseEnv, + final Consumer handleFailure) { + TestUtils.executeNonQueryWithRetry(baseEnv, "flush"); + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + getQueryCountSql(table), + "_col0,", + Collections.singleton(count + ","), + database, + handleFailure); + } + + public static void assertCountData( + final String database, + final String table, + final int count, + final BaseEnv baseEnv, + final DataNodeWrapper wrapper) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + wrapper, + getQueryCountSql(table), + "_col0,", + Collections.singleton(count + ","), + database); + } + + public static String getDateStr(final int value) { + Date date = new Date(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try { + return dateFormat.format(date); + } catch (Exception e) { + return "1970-01-01"; + } + } + + public static LocalDate getDate(final int value) { + Date date = new Date(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try { + return DateUtils.parseIntToLocalDate( + DateUtils.parseDateExpressionToInt(dateFormat.format(date))); + } catch (Exception e) { + return DateUtils.parseIntToLocalDate(DateUtils.parseDateExpressionToInt("1970-01-01")); + } + } + + public static Tablet generateTablet( + final String tableName, + final int start, + final int end, + final boolean allowNullValue, + final boolean allowNullDeviceColumn) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + end - start); + tablet.initBitMaps(); + Random random = new Random(); + int nullDeviceIndex = allowNullDeviceColumn ? random.nextInt(4) : 4; + + for (long row = 0; row < end - start; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + long value = start + row; + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, value); + tablet.addValue( + "s0", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s1", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s2", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s3", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (value * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, value); + tablet.addValue("s8", rowIndex, (int) value); + tablet.addValue("s9", rowIndex, value * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) value)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 11) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + if (nullDeviceIndex < 4) { + tablet.addValue("s" + nullDeviceIndex, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + + return tablet; + } + + public static Tablet generateTablet( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int start, + final int end, + final boolean allowNullValue, + final boolean allowNullDeviceColumn) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + (deviceEndIndex - deviceStartIndex) * (end - start)); + tablet.initBitMaps(); + final Random random = new Random(); + int nullDeviceIndex = allowNullDeviceColumn ? random.nextInt(4) : 4; + + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; deviceIndex++) { + for (long row = start; row < end; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, row); + tablet.addValue( + "s0", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s1", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s2", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s3", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, row); + tablet.addValue("s5", rowIndex, (row * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, row); + tablet.addValue("s8", rowIndex, (int) row); + tablet.addValue("s9", rowIndex, row * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) row)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 12) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + if (nullDeviceIndex < 4) { + tablet.addValue("s" + nullDeviceIndex, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + } + + return tablet; + } + + public static Tablet generateTablet( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int deviceDataSize, + final boolean allowNullValue, + final boolean allowNullDeviceColumn) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + (deviceEndIndex - deviceStartIndex) * deviceDataSize); + tablet.initBitMaps(); + final Random random = new Random(); + int nullDeviceIndex = allowNullDeviceColumn ? random.nextInt(4) : 4; + + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; deviceIndex++) { + // s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text + long value = random.nextInt(1 << 16); + for (long row = 0; row < deviceDataSize; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + int rowIndex = tablet.getRowSize(); + value += random.nextInt(100); + tablet.addTimestamp(rowIndex, value); + tablet.addValue( + "s0", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s1", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s2", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s3", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (value * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, value); + tablet.addValue("s8", rowIndex, (int) value); + tablet.addValue("s9", rowIndex, value * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) value)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 12) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + if (nullDeviceIndex < 4) { + tablet.addValue("s" + nullDeviceIndex, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + } + + return tablet; + } + + public static Tablet generateTabletDeviceIDAllIsNull( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int deviceDataSize, + final boolean allowNullValue) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + (deviceEndIndex - deviceStartIndex) * deviceDataSize); + tablet.initBitMaps(); + final Random random = new Random(); + + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; deviceIndex++) { + // s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text + long value = random.nextInt(1 << 16); + for (long row = 0; row < deviceDataSize; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + int rowIndex = tablet.getRowSize(); + value += random.nextInt(100); + tablet.addTimestamp(rowIndex, value); + tablet.addValue("s0", rowIndex, null); + tablet.addValue("s1", rowIndex, null); + tablet.addValue("s2", rowIndex, null); + tablet.addValue("s3", rowIndex, null); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (value * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, value); + tablet.addValue("s8", rowIndex, (int) value); + tablet.addValue("s9", rowIndex, value * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) value)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 12) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + } + + return tablet; + } + + public static int showPipesCount(final BaseEnv baseEnv, final String sqlDialect) { + try (final Connection connection = baseEnv.getConnection(sqlDialect); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + return count; + } catch (final SQLException e) { + fail(e.getMessage()); + } + return 0; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java new file mode 100644 index 0000000000000..fff0b8c6bf1a4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual; + +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.After; +import org.junit.Before; + +public abstract class AbstractPipeTableModelDualManualIT { + + protected BaseEnv senderEnv; + protected BaseEnv receiverEnv; + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + setupConfig(); + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + protected void setupConfig() { + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + } + + @After + public final void tearDown() { + senderEnv.cleanClusterEnvironment(); + receiverEnv.cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java new file mode 100644 index 0000000000000..61b29c30cc27d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java @@ -0,0 +1,489 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeAlterIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testBasicAlterPipe() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create pipe + // The database & table name will be converted to lower case + final String sql = + String.format( + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='Test', 'table-name'='Test1', 'mode.streaming'='true') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + long lastCreationTime; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Record last creation time + lastCreationTime = showPipeResult.get(0).creationTime; + } + + // Stop pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("stop pipe a2b"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + } + + // Alter pipe (modify) + // The database & table name will be converted to lower case + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b modify source ('table-name'='Test1','database-name'='Test1')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test1")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test1")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b replace source ('capture.table'='true','''source'='iotdb-source', 'table-name'='test','database-name'='test')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('sink.batch.enable'='false')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Start pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("start pipe a2b"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Alter pipe (modify) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('connector.batch.enable'='true')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b replace source ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("__system.sql-dialect=table")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b replace processor ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertFalse( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + } + + @Test + public void testAlterPipeFailure() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // alter non-existed pipe + String sql = + String.format( + "alter pipe a2b modify sink ('node-urls'='%s', 'batch.enable'='true')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + fail(); + } catch (SQLException ignore) { + } + + // Create pipe + sql = + String.format( + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test1', 'mode.streaming'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testAlterPipeSourceAndSink() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + // Create pipe + final String sql = + String.format( + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test', 'mode.streaming'='true') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s', 'batch.enable'='false')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + // Check data on receiver + TableModelUtils.assertData("test", "test", 0, 100, receiverEnv, handleFailure); + + // Alter pipe (modify 'source.path', 'source.inclusion' and + // 'processor.tumbling-time.interval-seconds') + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b modify source('source' = 'iotdb-source','database-name'='test1', 'table-name'='test1', 'mode.streaming'='true', 'source.inclusion'='data.insert') modify sink ('batch.enable'='true')"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 100), + "test", + handleFailure); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test1"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 200), + "test1", + handleFailure); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java new file mode 100644 index 0000000000000..23412f05edb23 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java @@ -0,0 +1,809 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "log"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "true"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.realtime-first", "false"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TableModelUtils.insertData("test", "test", 50, 100, senderEnv, true); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,"))), + handleFailure); + + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + } + } + + @Test + public void testSinkTabletFormat() throws Exception { + testSinkFormat("tablet"); + } + + @Test + public void testSinkTsFileFormat() throws Exception { + testSinkFormat("tsfile"); + } + + @Test + public void testSinkHybridFormat() throws Exception { + testSinkFormat("hybrid"); + } + + private void testSinkFormat(final String format) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "true"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.format", format); + connectorAttributes.put("connector.realtime-first", "false"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TableModelUtils.insertData("test", "test", 50, 150, senderEnv, true); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"))) { + return; + } + + TableModelUtils.assertCountData("test", "test", 150, receiverEnv, handleFailure); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))), + handleFailure); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush"))) { + return; + } + + TableModelUtils.insertData("test", "test", 150, 200, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 200, 250, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 250, 300, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 300, 350, senderEnv, true); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), + handleFailure); + + TableModelUtils.assertCountData("test", "test", 350, receiverEnv, handleFailure); + } + } + + @Test + public void testWriteBackSink() throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("forwarding-pipe-requests", "false"); + extractorAttributes.put("extractor.database-name", "test.*"); + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "rename-database-processor"); + processorAttributes.put("processor.new-db-name", "Test1"); + + connectorAttributes.put("connector", "write-back-sink"); + connectorAttributes.put("user", "root"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertDataNotThrowError("test", "test", 0, 20, senderEnv); + + TableModelUtils.insertTablet("test", "test", 20, 200, senderEnv, true); + + TableModelUtils.insertTablet("test", "test", 200, 400, senderEnv, true); + + TableModelUtils.assertCountData("test1", "test", 400, senderEnv); + } + } + + @Test + public void testSinkTsFileFormat2() throws Exception { + doTest(this::insertTablet1); + } + + @Test + public void testSinkTsFileFormat3() throws Exception { + doTest(this::insertTablet2); + } + + @Test + public void testSinkTsFileFormat4() throws Exception { + doTest(this::insertTablet3); + } + + @Test + public void testSinkTsFileFormat5() throws Exception { + doTest(this::insertTablet4); + } + + @Test + public void testSinkTsFileFormat6() throws Exception { + doTest(this::insertTablet5); + } + + @Test + public void testSinkTsFileFormat7() throws Exception { + doTest(this::insertTablet6); + } + + @Test + public void testSinkTsFileFormat8() throws Exception { + doTest(this::insertTablet7); + } + + @Test + public void testSinkTsFileFormat9() throws Exception { + doTest(this::insertTablet8); + } + + @Test + public void testSinkTsFileFormat10() throws Exception { + doTest(this::insertTablet9); + } + + private void doTest(BiConsumer>, Map>> consumer) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + Map> testResult = new HashMap<>(); + Map> test1Result = new HashMap<>(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + for (int i = 0; i < 5; i++) { + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test0"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test1"); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("extractor.database-name", "test.*"); + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.format", "tsfile"); + connectorAttributes.put("connector.realtime-first", "true"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"))) { + return; + } + + consumer.accept(testResult, test1Result); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))), + handleFailure); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), + handleFailure); + } + + for (Map.Entry> entry : testResult.entrySet()) { + final Set set = new HashSet<>(); + entry + .getValue() + .forEach( + tablet -> { + set.addAll(TableModelUtils.generateExpectedResults(tablet)); + }); + TableModelUtils.assertCountData( + "test0", entry.getKey(), set.size(), receiverEnv, handleFailure); + TableModelUtils.assertData("test0", entry.getKey(), set, receiverEnv, handleFailure); + } + + for (Map.Entry> entry : test1Result.entrySet()) { + final Set set = new HashSet<>(); + entry + .getValue() + .forEach( + tablet -> { + set.addAll(TableModelUtils.generateExpectedResults(tablet)); + }); + TableModelUtils.assertCountData( + "test1", entry.getKey(), set.size(), receiverEnv, handleFailure); + TableModelUtils.assertData("test1", entry.getKey(), set, receiverEnv, handleFailure); + } + } + + private void insertTablet1( + final Map> testResult, final Map> test1Result) { + + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 10; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 0, 10, false, false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet2( + final Map> testResult, final Map> test1Result) { + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 10; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 10, false, false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet3( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + + for (int j = 0; j < 25; j++) { + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + final String dataBaseName = "test" + j % 2; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, + deviceIDStartIndex, + deviceIDEndIndex, + deviceIDStartIndex, + deviceIDEndIndex, + false, + true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet4( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, + deviceIDStartIndex, + deviceIDEndIndex, + deviceIDStartIndex, + deviceIDEndIndex, + false, + false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet5( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + for (int j = 0; j < 25; j++) { + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + final String dataBaseName = "test" + j % 2; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, 0, 10, deviceIDStartIndex, deviceIDEndIndex, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet6( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 100, 110, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet7( + final Map> testResult, final Map> test1Result) { + + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 100, 110, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet8( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = random.nextInt(1 << 16); + int deviceIDEndIndex = deviceIDStartIndex + 10; + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, 100, 110, deviceIDStartIndex, deviceIDEndIndex, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet9( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTabletDeviceIDAllIsNull(tableName, 100, 110, 10, false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + @Test + public void testLoadTsFileWithoutVerify() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + for (int i = 0; i < 5; i++) { + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test0"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test1"); + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "batch"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + connectorAttributes.put("sink.tsfile.validation", "false"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create timeSeries root.vehicle.d0.s1 int32", + "insert into root.vehicle.d0(time, s1) values (2, 1)", + "flush"))) { + return; + } + + Map> testResult = new HashMap<>(); + Map> test1Result = new HashMap<>(); + + insertTablet1(testResult, test1Result); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + + for (Map.Entry> entry : testResult.entrySet()) { + final Set set = new HashSet<>(); + entry + .getValue() + .forEach( + tablet -> { + set.addAll(TableModelUtils.generateExpectedResults(tablet)); + }); + TableModelUtils.assertCountData("test0", entry.getKey(), set.size(), receiverEnv, s -> {}); + TableModelUtils.assertData("test0", entry.getKey(), set, receiverEnv, s -> {}); + } + + for (Map.Entry> entry : test1Result.entrySet()) { + final Set set = new HashSet<>(); + entry + .getValue() + .forEach( + tablet -> { + set.addAll(TableModelUtils.generateExpectedResults(tablet)); + }); + TableModelUtils.assertCountData("test1", entry.getKey(), set.size(), receiverEnv, s -> {}); + TableModelUtils.assertData("test1", entry.getKey(), set, receiverEnv, s -> {}); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java new file mode 100644 index 0000000000000..d45e924620f04 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java @@ -0,0 +1,985 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeExtractorIT extends AbstractPipeTableModelDualManualIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Disable sender compaction for tsfile determination in loose range test + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testMatchingMultipleDatabases() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "test"); + extractorAttributes.put("extractor.pattern", "root.db1"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + assertTimeseriesCountOnReceiver(receiverEnv, 0); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1) values (1, 10)", + "insert into root.db2.d1 (time, at1) values (1, 20)", + "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + extractorAttributes.replace("extractor.pattern", "root.db2"); + extractorAttributes.replace("extractor.table-name", "test1"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + assertTimeseriesCountOnReceiver(receiverEnv, 2); + + Thread.sleep(10000); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1) values (2, 11)", + "insert into root.db2.d1 (time, at1) values (2, 21)", + "flush"))) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases + extractorAttributes.remove("extractor.table-name"); + extractorAttributes.remove("extractor.database-name"); + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db1.d1.at1),count(root.db2.d1.at1),", + Collections.singleton("2,2,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + } + } + + @Test + public void testHistoryAndRealtime() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (1, 10)", + "insert into root.db.d2 (time, at1) values (1, 20)", + "insert into root.db.d3 (time, at1) values (1, 30)", + "insert into root.db.d4 (time, at1) values (1, 40)", + "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test3"); + insertResult = TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); + insertResult = TableModelUtils.insertData("test", "test4", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.database", "test"); + extractorAttributes.put("extractor.table", "test2"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.pattern", "root.db.d2"); + extractorAttributes.put("extractor.history.enable", "false"); + extractorAttributes.put("extractor.realtime.enable", "true"); + extractorAttributes.put("user", "root"); + TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + extractorAttributes.replace("extractor.table-name", "test3"); + extractorAttributes.replace("extractor.pattern", "root.db.d3"); + extractorAttributes.replace("extractor.history.enable", "true"); + extractorAttributes.replace("extractor.realtime.enable", "false"); + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + + extractorAttributes.replace("extractor.table-name", "test4"); + extractorAttributes.replace("extractor.pattern", "root.db.d4"); + extractorAttributes.replace("extractor.history.enable", "true"); + extractorAttributes.replace("extractor.realtime.enable", "true"); + status = + client.createPipe( + new TCreatePipeReq("p4", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p4").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (2, 11)", + "insert into root.db.d2 (time, at1) values (2, 21)", + "insert into root.db.d3 (time, at1) values (2, 31)", + "insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"))) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test2", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test4", 0, 200, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** where time <= 1", + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + Collections.singleton("1,0,1,"), + handleFailure); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** where time >= 2", + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + Collections.singleton("2,1,0,"), + handleFailure); + + TableModelUtils.assertData("test", "test2", 100, 200, receiverEnv, handleFailure); + + TableModelUtils.assertData("test", "test3", 100, 200, receiverEnv, handleFailure); + + TableModelUtils.assertData("test", "test4", 100, 200, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "insert into root.db.d2 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); + if (!insertResult) { + return; + } + + // wait for flush to complete + if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList("flush"))) { + return; + } + Thread.sleep(10000); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "test1"); + extractorAttributes.put("extractor.pattern", "root.db.d1"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.history.enable", "true"); + // 1970-01-01T08:00:02+08:00 + extractorAttributes.put("extractor.history.start-time", "2"); + extractorAttributes.put("extractor.history.end-time", "3"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),", + Collections.singleton("2,"), + handleFailure); + + extractorAttributes.remove("extractor.table-name"); + extractorAttributes.remove("extractor.pattern"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d2.at1),", + Collections.singleton("2,2,"), + handleFailure); + + TableModelUtils.assertData("test", "test1", 2, 4, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test2", 2, 4, receiverEnv, handleFailure); + } + } + + @Test + public void testExtractorTimeRangeMatch() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // insert history data + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "insert into root.db.d2 (time, at1)" + + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", + "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.start-time", "2"); + extractorAttributes.put("source.end-time", "4"); + extractorAttributes.put("user", "root"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),", + Collections.singleton("3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + + // Insert realtime data that overlapped with time range + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d3 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "flush"))) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); + insertResult = TableModelUtils.insertData("test", "test3", 0, 5, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d3.at1),", + Collections.singleton("3,3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); + + // Insert realtime data that does not overlap with time range + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d4 (time, at1)" + + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", + "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); + insertResult = TableModelUtils.insertData("test", "test4", 6, 10, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d3.at1),", + Collections.singleton("3,3,"), + 600, + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); + } + } + + @Test + public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "insert into root.db.d2 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "flush"))) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 5, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 5, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("source.pattern", "root.db.d1"); + extractorAttributes.put("source.table-name", "test1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.start-time", "2"); + // 1970-01-01T08:00:04+08:00 + extractorAttributes.put("source.end-time", "4"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),", + Collections.singleton("3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + + extractorAttributes.remove("source.pattern"); + extractorAttributes.remove("source.table-name"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d2.at1),", + Collections.singleton("3,3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test2", 3, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testHistoryLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 2, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 2, senderEnv); + if (!insertResult) { + return; + } + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 1, extracted without parse + "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", + // TsFile 2, not extracted because pattern not overlapped + "insert into root.db1.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", + "flush"))) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 3, not extracted because time range not overlapped + "insert into root.db.d1 (time, at1, at2)" + " values (3, 1, 2), (4, 3, 4)", + "flush"))) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test1", 2, 5, senderEnv); + if (!insertResult) { + return; + } + + // wait for flush to complete + if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList("flush"))) { + return; + } + Thread.sleep(10000); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("source.table-name", "test1"); + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.history.start-time", "1"); + extractorAttributes.put("source.history.end-time", "2"); + extractorAttributes.put("source.history.loose-range", "time, path"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** group by level=0", + "count(root.*.*.*),", + Collections.singleton("4,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testRealtimeLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + insertResult = TableModelUtils.insertData("test", "test2", 0, 20, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("source.table-name", "test1"); + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.realtime.loose-range", "time, path"); + extractorAttributes.put("source.start-time", "2"); + extractorAttributes.put("source.end-time", "10"); + extractorAttributes.put("source.realtime.mode", "batch"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (3, 3, 4)", + "flush"))) { + return; + } + + if (!insertResult) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + " values (5, 1), (16, 3)", + "insert into root.db.d1 (time, at1, at2)" + " values (5, 1, 2), (6, 3, 4)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(at1) from root.db.d1 where time >= 2 and time <= 10", + new HashMap() { + { + put("count(root.db.d1.at1)", "3"); + } + }); + + insertResult = TableModelUtils.insertData("test", "test1", 10, 20, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test2", 10, 20, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertCountData("test", "test1", 20, receiverEnv, handleFailure); + } + } + + @Test + public void testTableModeSQLSupportNowFunc() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final String p1 = + String.format( + "create pipe p1" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(p1); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + final String p2 = + String.format( + "create pipe p2" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(p2); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + final String p3 = + String.format( + "create pipe p3" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(p3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + String alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + alterP3 = + "alter pipe p3" + + " modify source (" + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + } + + private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton(count + ",")); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java new file mode 100644 index 0000000000000..6924f17cedc43 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testWritePipeIsolation() throws Exception { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Start pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(true)) + .getCode()); + + // Stop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + + // Alter pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(true)) + .getCode()); + + // Drop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + } + } + + @Test + public void testReadPipeIsolation() { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } + + @Test + public void testCaptureTreeAndTableIsolation() throws Exception { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Drop pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } + + @Test + public void testCaptureCornerCases() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe but capture table data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe but capture tree data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Create pipe with capture.tree and capture.table set to false + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java new file mode 100644 index 0000000000000..af11dd3b31ebe --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java @@ -0,0 +1,787 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testLifeCycleWithHistoryEnabled() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 300, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 400, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 400, 500, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 500, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testLifeCycleWithHistoryDisabled() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + // wait for flush to complete + if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList("flush"))) { + return; + } + Thread.sleep(10000); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.inclusion.exclusion", ""); + + extractorAttributes.put("extractor.history.enable", "false"); + // start-time and end-time should not work + extractorAttributes.put("extractor.history.start-time", "0"); + extractorAttributes.put("extractor.history.end-time", "50"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertCountData("test1", "test1", 100, receiverEnv, handleFailure); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test2"); + insertResult = TableModelUtils.insertData("test2", "test2", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test1", "test1", 100, receiverEnv, handleFailure); + } + } + + @Test + public void testLifeCycleLogMode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.mode", "forced-log"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test1", "test1", 100, receiverEnv, handleFailure); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test2"); + insertResult = TableModelUtils.insertData("test2", "test2", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test2", "test2", 100, receiverEnv, handleFailure); + } + } + + @Test + public void testLifeCycleFileMode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("mode.streaming", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test1", "test1", 100, receiverEnv, handleFailure); + } + } + + @Test + public void testLifeCycleHybridMode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.mode", "hybrid"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 300, receiverEnv, handleFailure); + } + } + + @Test + public void testLifeCycleWithClusterRestart() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient ignored = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 300, receiverEnv, handleFailure); + } + } + + @Test + public void testReceiverRestartWhenTransferring() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final AtomicInteger succeedNum = new AtomicInteger(0); + final Thread t = + new Thread( + () -> { + try { + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError( + "test", "test", i, i + 1, senderEnv)) { + succeedNum.incrementAndGet(); + } + Thread.sleep(100); + } + } catch (InterruptedException ignored) { + } + }); + t.start(); + + try { + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + try { + t.interrupt(); + t.join(); + } catch (Throwable ignored) { + } + return; + } + + t.join(); + client.stopPipe("p1"); + client.startPipe("p1"); + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + TableModelUtils.assertCountData( + "test", "test", 100 + succeedNum.get(), receiverEnv, handleFailure); + } + } + + @Test + public void testReceiverAlreadyHaveTimeSeries() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(receiverEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + } + } + + @Test + public void testDoubleLiving() throws Exception { + boolean insertResult = true; + // Double living is two clusters with pipes connecting each other. + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Consumer handleFailure = + o -> { + executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + }; + + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // Add this property to avoid to make self cycle. + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("forwarding-pipe-requests", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, receiverEnv); + if (!insertResult) { + return; + } + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // Add this property to avoid to make self cycle. + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("forwarding-pipe-requests", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + insertResult = TableModelUtils.insertData("test", "test", 300, 400, receiverEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + insertResult = TableModelUtils.insertData("test", "test", 400, 500, receiverEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); + } + + @Test + public void testPermission() { + createUser(senderEnv, "test", "test123"); + + assertTableNonQueryTestFail( + senderEnv, + "create pipe testPipe\n" + + "with connector (\n" + + " 'connector'='iotdb-thrift-connector',\n" + + " 'connector.ip'='127.0.0.1',\n" + + " 'connector.port'='6668'\n" + + ")", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "drop pipe testPipe", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableTestFail( + senderEnv, + "show pipes", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "start pipe testPipe", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "stop pipe testPipe", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + + assertTableNonQueryTestFail( + senderEnv, + "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "drop pipePlugin TestProcessor", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableTestFail( + senderEnv, + "show pipe plugins", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java new file mode 100644 index 0000000000000..890a90cdb5316 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipePermissionIT extends AbstractPipeTableModelDualManualIT { + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setDefaultSchemaRegionGroupNumPerDatabase(1) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(3, 3); + } + + @Test + public void testSourcePermission() { + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "create user `thulab` 'passwd'")) { + return; + } + + // Shall fail if username is specified without password + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab'" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail("When the 'user' or 'username' is specified, password must be specified too."); + } catch (final SQLException ignore) { + // Expected + } + + // Shall fail if password is wrong + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab'" + + "'password'='hack'" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail("Shall fail if password is wrong."); + } catch (final SQLException ignore) { + // Expected + } + + // Use current session, user is root + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'inclusion'='all'," + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Create pipe without user shall succeed if use the current session"); + } + + // Alter to another user, shall fail because of lack of password + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('username'='thulab')"); + fail("Alter pipe shall fail if only user is specified"); + } catch (final SQLException ignore) { + // Expected + } + + // Successfully alter + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('username'='thulab', 'password'='passwd')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Alter pipe shall not fail if user and password are specified"); + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "count databases", + "count,", + Collections.singleton("1,"), + "information_schema"); + + // Grant some privilege + if (!TestUtils.tryExecuteNonQueryWithRetry( + "test", BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab")) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + + // Shall not be transferred + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show tables from test1", + "TableName,TTL(ms),", + Collections.singleton("test1,INF,"), + "information_schema"); + + // Alter pipe, throw exception if no privileges + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('skipif'='')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Write some data + if (!TableModelUtils.insertData("test", "test", 0, 100, senderEnv)) { + return; + } + + TableModelUtils.createDataBaseAndTable(receiverEnv, "test", "test"); + + // Exception, block here + TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, receiverEnv); + + // Grant SELECT privilege + if (!TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + Arrays.asList("grant SELECT on any to user thulab", "start pipe a2b"))) { + return; + } + + // Will finally pass + TableModelUtils.assertCountData( + "test", + "test", + 100, + receiverEnv, + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }); + } + + @Test + public void testReceiverPermission() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "create user testUser 'password'")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + final String dbName = "test"; + final String tbName = "test"; + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "testUser"); + connectorAttributes.put("connector.password", "password"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.startPipeExtended(new TStartPipeReq("testPipe").setIsTableModel(true)).getCode()); + + TableModelUtils.createDataBaseAndTable(senderEnv, tbName, dbName); + + // Write some data + if (!TableModelUtils.insertData(dbName, tbName, 0, 100, senderEnv)) { + return; + } + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show databases", + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,"), + (String) null); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + "information_schema", + BaseEnv.TABLE_SQL_DIALECT, + receiverEnv, + "grant insert,create on database test to user testUser")) { + return; + } + + // Will finally pass + TableModelUtils.assertCountData( + dbName, + tbName, + 100, + receiverEnv, + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }); + + // Alter pipe, skip if no privileges + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe testPipe modify sink ('skipif'='no-privileges')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final String dbName2 = "test2"; + + // Write some data + if (!TableModelUtils.insertData(dbName2, tbName, 0, 100, senderEnv)) { + return; + } + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("2,"), (String) null); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + "information_schema", + BaseEnv.TABLE_SQL_DIALECT, + receiverEnv, + "grant insert,create on database test2 to user testUser")) { + return; + } + + if (!TableModelUtils.insertData(dbName2, tbName, 100, 200, senderEnv)) { + return; + } + + // Will finally pass + TableModelUtils.assertCountData( + dbName2, + tbName, + 100, + receiverEnv, + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java new file mode 100644 index 0000000000000..4e18091bd07e8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java @@ -0,0 +1,492 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + } + + private void innerSetUp( + final String configNodeConsensus, + final String schemaRegionConsensus, + final String dataRegionConsensus, + final int configNodesNum, + final int dataNodesNum, + int schemaRegionReplicationFactor, + int dataRegionReplicationFactor) { + schemaRegionReplicationFactor = Math.min(schemaRegionReplicationFactor, dataNodesNum); + dataRegionReplicationFactor = Math.min(dataRegionReplicationFactor, dataNodesNum); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(dataRegionConsensus) + .setSchemaReplicationFactor(schemaRegionReplicationFactor) + .setDataReplicationFactor(dataRegionReplicationFactor); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(configNodeConsensus) + .setSchemaRegionConsensusProtocolClass(schemaRegionConsensus) + .setDataRegionConsensusProtocolClass(dataRegionConsensus) + .setSchemaReplicationFactor(schemaRegionReplicationFactor) + .setDataReplicationFactor(dataRegionReplicationFactor); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + } + + @Test + public void test1C1DWithRatisRatisIot() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test1C1DWithSimpleSimpleIot() throws Exception { + innerSetUp( + ConsensusFactory.SIMPLE_CONSENSUS, + ConsensusFactory.SIMPLE_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test1C1DWithRatisRatisSimple() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.SIMPLE_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 3, + 3, + 3, + 3); + doTest(); + } + + @Test + public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 3, + 3, + 3, + 2); + doTest(); + } + + @Test + public void testPipeOnBothSenderAndReceiver() throws Exception { + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(3, 3); + receiverEnv.initClusterEnvironment(1, 1); + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + } + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, receiverEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test.*"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertCountData("test1", "test1", 100, senderEnv, handleFailure); + } + } + + private void doTest() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test.*"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertData("test", "test", 0, 100, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + } + } + + @Test + public void testSyncConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); + } + + @Test + public void testAsyncConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); + } + + @Test + public void testAirGapConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); + } + + private void doTestUseNodeUrls(String connectorName) throws Exception { + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1) + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(1, 1); + receiverEnv.initClusterEnvironment(1, 3); + + final StringBuilder nodeUrlsBuilder = new StringBuilder(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { + if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { + // Use default port for convenience + nodeUrlsBuilder + .append(wrapper.getIp()) + .append(":") + .append(wrapper.getPipeAirGapReceiverPort()) + .append(","); + } else { + nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); + } + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", connectorName); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.batch.max-delay-seconds", "1"); + connectorAttributes.put("connector.batch.size-bytes", "2048"); + connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test.*"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + // Test forced-log mode, in open releases this might be "file" + extractorAttributes.put("realtime.mode", "forced-log"); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + + extractorAttributes.replace("realtime.mode", "file"); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java new file mode 100644 index 0000000000000..50fb130a81257 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testPipeSwitchStatus() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("start-time", "1"); + extractorAttributes.put("end-time", "2"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p2") && o.state.equals("RUNNING"))); + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p3") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p3").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p2") && o.state.equals("RUNNING"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + } + } + + @Test + public void testPipeIllegallySwitchStatus() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("start-time", "1"); + extractorAttributes.put("end-time", "2"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), status.getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + } + } + + @Test + public void testDropPipeAndCreateAgain() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "200"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals( + 0, + showPipeResult.stream() + .filter(info -> !info.id.startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) + .count()); + + status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + } + } + + @Test + public void testWrongPipeName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "200"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("*").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java new file mode 100644 index 0000000000000..b7202a9d0d1d1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java @@ -0,0 +1,706 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testValidPipeName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final List validPipeNames = + Arrays.asList("Pipe_1", "\"`33`\"", "root", "\"`中文`\"", "\"`with`\""); + final List expectedPipeNames = + Arrays.asList("Pipe_1", "`33`", "root", "`中文`", "`with`"); + for (final String pipeName : validPipeNames) { + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source ( " + + "'capture.table'='true'," + + "'database-name'='test'," + + "'table-name'='test'," + + "'mode.streaming'='true'," + + "'mode.strict'='true'," + + "'mode.snapshot'='false'," + + "'start-time'='1'," + + "'end-time'='2')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + pipeName, receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + for (final String pipeName : expectedPipeNames) { + Assert.assertTrue( + showPipeResult.stream() + .anyMatch((o) -> o.id.equals(pipeName) && o.state.equals("RUNNING"))); + } + + for (final String pipeName : validPipeNames) { + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("drop pipe %s", pipeName)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + showPipeResult = client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testRevertParameterOrder() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with source ( " + + "'capture.table'='true'," + + "'database-name'='test'," + + "'table-name'='test'," + + "'mode.streaming'='true'," + + "'mode.strict'='true'," + + "'mode.snapshot'='false'," + + "'start-time'='1'," + + "'end-time'='2')" + + " with connector (" + + "'connector.batch.enable'='false', " + + "'connector.port'='%s'," + + "'connector.ip'='%s'," + + "'connector'='iotdb-thrift-connector')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignore) { + } + } + + @Test + public void testRevertStageOrder() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with connector (" + + "'connector.batch.enable'='false', " + + "'connector.port'='%s'," + + "'connector.ip'='%s'," + + "'connector'='iotdb-thrift-connector') " + + " with source ( " + + "'capture.table'='true'," + + "'database-name'='test'," + + "'table-name'='test'," + + "'mode.streaming'='true'," + + "'mode.strict'='true'," + + "'mode.snapshot'='false'," + + "'start-time'='1'," + + "'end-time'='2')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testMissingStage() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create pipe p1"); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create pipe p2 with extractor ('extractor'='iotdb-extractor')"); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create pipe p3" + + " with extractor ('extractor'='iotdb-extractor')" + + " with processor ('processor'='do-nothing-processor')"); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p5" + + " with extractor ('extractor'='iotdb-extractor')" + + " with processor ('processor'='do-nothing-processor')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(2, showPipeResult.size()); + } + } + + @Test + public void testInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with extractor ()" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p3" + + " with extractor ()" + + " with processor ('processor'='invalid-param')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ()" + + " with processor ()" + + " with connector (" + + "'connector'='invalid-param'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + } + } + + @Test + public void testBrackets() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor1" + + " with extractor ('extractor'='iotdb-extractor')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe source2" + + " with extractor (\"extractor\"=\"iotdb-extractor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException e) { + e.printStackTrace(); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor3" + + " with extractor (" + + "'capture.table'='true'," + + "'database-name'='test'," + + "'table-name'='test'," + + "'mode.streaming'='true'," + + "'mode.strict'='true'," + + "'mode.snapshot'='false'," + + "'start-time'='1'," + + "'end-time'='2'," + + "'extractor'=\"iotdb-extractor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException e) { + e.printStackTrace(); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor4" + + " with extractor (" + + "'extractor'=iotdb-extractor," + + "'capture.table'='true'," + + "'database-name'='test'," + + "'table-name'='test'," + + "'mode.streaming'='true'," + + "'mode.strict'='true'," + + "'mode.snapshot'='false'," + + "'start-time'='1'," + + "'end-time'='2')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor5" + + " with extractor (" + + "'extractor'=`iotdb-extractor`," + + "'capture.table'='true'," + + "'database-name'='test'," + + "'table-name'='test'," + + "'mode.streaming'='true'," + + "'mode.strict'='true'," + + "'mode.snapshot'='false'," + + "'start-time'='1'," + + "'end-time'='2')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor1" + + " with processor ('processor'='do-nothing-processor')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor2" + + " with processor (\"processor\"=\"do-nothing-processor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (Exception e) { + e.printStackTrace(); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor3" + + " with processor ('processor'=\"do-nothing-processor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (Exception e) { + e.printStackTrace(); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor4" + + " with processor (processor=do-nothing-processor)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (Exception ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor5" + + " with processor ('processor'=`do-nothing-processor`)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (Exception ignored) { + } + + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(2, showPipeResult.size()); + } + } + + @Test + public void testShowPipeWithWrongPipeName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "table"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.mode.streaming", "true"); + extractorAttributes.put("extractor.mode.snapshot", "true"); + extractorAttributes.put("extractor.mode.strict", "true"); + extractorAttributes.put("extractor.start-time", "1"); + extractorAttributes.put("extractor.end-time", "2"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + connectorAttributes.replace("connector.batch.enable", "true"); + + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(3, showPipeResult.size()); + + showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true).setPipeName("p1")).pipeInfoList; + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); + + // Show all pipes whose connector is also used by p1. + // p1 and p2 share the same connector parameters, so they have the same connector. + showPipeResult = + client.showPipe( + new TShowPipeReq().setIsTableModel(true).setPipeName("p1").setWhereClause(true)) + .pipeInfoList; + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); + } + } + + @Test + public void testInclusionPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Empty inclusion + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p2" + + " with extractor ('extractor.inclusion'='schema, auth.role', 'extractor.inclusion.exclusion'='all')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (final SQLException ignored) { + } + + // Invalid inclusion + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p3" + + " with extractor ('extractor.inclusion'='wrong')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + // Invalid exclusion + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.inclusion.exclusion'='wrong')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + // Valid + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.inclusion'='all', 'extractor.inclusion.exclusion'='schema, auth')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java new file mode 100644 index 0000000000000..bc4e7f0205328 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java @@ -0,0 +1,490 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelDualManualIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Disable sender compaction to test mods + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Ignore // not support + @Test + public void testReceiverNotLoadDeletedTimeseries() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // Enable mods transfer + extractorAttributes.put("mods", "true"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Generate TsFile + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.deleteData("test", "test", 50, 100, senderEnv); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertCountData("test", "test", 50, receiverEnv, handleFailure); + } + } + + // Test that receiver will not load data when table exists but TAG columns mismatch + @Test + public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(2, 'd1', 'd2', 'blue', 2)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag3 STRING TAG, tag4 STRING TAG, s3 TEXT FIELD, s4 INT32 FIELD)"); + statement.execute("INSERT INTO t1(time,tag3,tag4,s3,s4) values(1, 'd3', 'd4', 'red2', 10)"); + statement.execute( + "INSERT INTO t1(time,tag3,tag4,s3,s4) values(2, 'd3', 'd4', 'blue2', 20)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + try { + // wait some time + Thread.sleep(10_000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + Set expectedResSet = new java.util.HashSet<>(); + expectedResSet.add("1970-01-01T00:00:00.002Z,d3,d4,blue2,20,null,null,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.001Z,d3,d4,red2,10,null,null,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,null,null,null,null,d1,d2,blue,2,"); + expectedResSet.add("1970-01-01T00:00:00.001Z,null,null,null,null,d1,d2,red,1,"); + // make sure data are not transferred + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from t1", + "time,tag3,tag4,s3,s4,tag1,tag2,s1,s2,", + expectedResSet, + "db", + handleFailure); + } + } + + // Test that receiver can load data when table exists and existing TAG columns are the prefix of + // incoming TAG columns + @Test + public void testReceiverAutoExtendIdColumn() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); + statement.execute( + "INSERT INTO t1(time,tag1,tag2,tag3,s1,s2) values(1, 'd1', 'd2', 'd3', 'red', 1)"); + statement.execute( + "INSERT INTO t1(time,tag1,tag2,tag3,s1,s2) values(2, 'd1', 'd2', 'd3', 'blue', 2)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s3 TEXT FIELD, s4 INT32 FIELD)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s3,s4) values(1, 'd1', 'd2', 'red2', 10)"); + statement.execute( + "INSERT INTO t1(time,tag1,tag2,s3,s4) values(2, 'd1', 'd2', 'blue2', 20)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + Set expectedResSet = new java.util.HashSet<>(); + expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,null,null,d3,red,1,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,d1,d2,null,null,d3,blue,2,"); + expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,red2,10,null,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,d1,d2,blue2,20,null,null,null,"); + // make sure data are transferred and column "tag3" is auto extended + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from t1", + "time,tag1,tag2,s3,s4,tag3,s1,s2,", + expectedResSet, + "db", + handleFailure); + } + } + + // Test that receiver can load data when table exists and incoming TAG columns are the prefix of + // existing TAG columns + @Test + public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(2, 'd1', 'd2', 'blue', 2)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG,s3 TEXT FIELD, s4 INT32 FIELD)"); + statement.execute( + "INSERT INTO t1(time,tag1,tag2,tag3,s3,s4) values(1, 'd1', 'd2', 'd3', 'red2', 10)"); + statement.execute( + "INSERT INTO t1(time,tag1,tag2,tag3,s3,s4) values(2, 'd1', 'd2', 'd3', 'blue2', 20)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + Set expectedResSet = new java.util.HashSet<>(); + expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,d3,red2,10,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,d1,d2,d3,blue2,20,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,null,null,null,red,1,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,d1,d2,null,null,null,blue,2,"); + // make sure data are transferred and column "tag3" is null in transferred data + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from t1", + "time,tag1,tag2,tag3,s3,s4,s1,s2,", + expectedResSet, + 10, + "db", + handleFailure); + } + } + + @Test + public void testLoadAutoCreateWithTableDeletion() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(2, 'd1', 'd2', 'blue', 2)"); + statement.execute("flush"); + statement.execute("drop table t1"); + } catch (Exception e) { + fail(e.getMessage()); + } + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + // Ensure the deleted table won't be created + // Now the database will also be created at receiver + TestUtils.assertAlwaysFail(receiverEnv, "describe db.t1"); + } + } + + @Test + public void testLoadAutoCreateWithoutInsertPermission() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "user01"); + connectorAttributes.put("connector.password", "1234"); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create user user01 '1234'"); + statement.execute("grant create on any to user user01"); + } catch (final Exception e) { + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db"); + statement.execute("use db"); + statement.execute( + "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); + statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(2, 'd1', 'd2', 'blue', 2)"); + statement.execute("flush"); + } catch (final Exception e) { + fail(e.getMessage()); + } + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + // Ensure the table without insert privilege won't be created + // Now the database will also be created at receiver + TestUtils.assertAlwaysFail(receiverEnv, "describe db.t1"); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java new file mode 100644 index 0000000000000..f89558325ef5c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java @@ -0,0 +1,618 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.function.Consumer; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testTableNamePattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData("test1", "test1", 0, 200, receiverEnv, handleFailure); + if (!TableModelUtils.hasDataBase("test", receiverEnv)) { + Assert.fail(); + } + if (!TableModelUtils.hasDataBase("test1", receiverEnv)) { + Assert.fail(); + } + } + } + + @Test + public void testTableNamePatternByHistoryTSFile() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // extractorAttributes.put("extractor.database-name", "test.*"); + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.start-time", "0"); + extractorAttributes.put("extractor.end-time", "49"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertData("test", "test", 0, 50, receiverEnv, handleFailure); + TableModelUtils.assertData("test1", "test1", 0, 50, receiverEnv, handleFailure); + if (!TableModelUtils.hasDataBase("test", receiverEnv)) { + Assert.fail(); + } + if (!TableModelUtils.hasDataBase("test1", receiverEnv)) { + Assert.fail(); + } + } + } + + @Test + public void testTableNamePatternByRealTime() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // extractorAttributes.put("extractor.database-name", "test.*"); + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.start-time", "100"); + extractorAttributes.put("extractor.end-time", "149"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 100, 150, receiverEnv, handleFailure); + TableModelUtils.assertData("test1", "test1", 100, 150, receiverEnv, handleFailure); + if (!TableModelUtils.hasDataBase("test", receiverEnv)) { + Assert.fail(); + } + if (!TableModelUtils.hasDataBase("test1", receiverEnv)) { + Assert.fail(); + } + } + } + + @Test + public void testDataBasePattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("pattern", "pattern", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData("pattern1", "pattern1", 0, 200, receiverEnv, handleFailure); + + HashSet expectedResults = new HashSet(); + if (!TableModelUtils.hasDataBase("pattern", receiverEnv)) { + Assert.fail(); + } + if (!TableModelUtils.hasDataBase("pattern1", receiverEnv)) { + Assert.fail(); + } + } + } + + @Test + public void testDataBasePatternByHistoryTSFile() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + TableModelUtils.assertData("pattern", "pattern", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData("pattern1", "pattern1", 0, 100, receiverEnv, handleFailure); + + if (!TableModelUtils.hasDataBase("pattern1", receiverEnv)) { + Assert.fail(); + } + if (!TableModelUtils.hasDataBase("pattern", receiverEnv)) { + Assert.fail(); + } + } + } + + @Test + public void testDataBasePatternByRealtime() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("pattern", "pattern", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData("pattern1", "pattern1", 0, 100, receiverEnv, handleFailure); + + if (!TableModelUtils.hasDataBase("pattern", receiverEnv)) { + Assert.fail(); + } + if (!TableModelUtils.hasDataBase("pattern1", receiverEnv)) { + Assert.fail(); + } + } + } + + @Test + public void testIoTDBPatternWithDataBaseAndTable() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.table-name", "test.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show databases", + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,"), + (String) null); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java new file mode 100644 index 0000000000000..500df873fd73f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelDualManualIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testDoubleLivingAutoConflict() throws Exception { + // Double living is two clusters each with a pipe connecting to the other. + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + createDataBaseAndTable(senderEnv); + createDataBaseAndTable(receiverEnv); + + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + insertData("test", "test1", 0, 100, senderEnv); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.inclusion.exclusion", ""); + extractorAttributes.put("capture.table", "true"); + // Add this property to avoid making self cycle. + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + insertData("test", "test1", 100, 200, senderEnv); + insertData("test", "test1", 200, 300, receiverEnv); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.inclusion.exclusion", ""); + extractorAttributes.put("capture.table", "true"); + // Add this property to avoid to make self cycle. + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + insertData("test", "test1", 300, 400, receiverEnv); + + TableModelUtils.assertData("test", "test1", 0, 400, receiverEnv, handleFailure); + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + fail(e.getMessage()); + } + + insertData("test", "test1", 400, 500, senderEnv); + insertData("test", "test1", 500, 600, receiverEnv); + + TableModelUtils.assertData("test", "test1", 0, 600, receiverEnv, handleFailure); + } + + @Test + public void testDoubleLivingAutoConflictTemplate() throws Exception { + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + createDataBaseAndTable(senderEnv); + createDataBaseAndTable(receiverEnv); + insertData("test", "test", 0, 100, senderEnv); + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.inclusion.exclusion", ""); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData("test", "test1", 200, 300, receiverEnv); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.inclusion.exclusion", ""); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "true"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + insertData("test", "test1", 300, 400, senderEnv); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 200), + "test", + handleFailure); + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + TableModelUtils.getQuerySql("test1"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(200, 400), + "test", + handleFailure); + } + + private void createDataBaseAndTable(BaseEnv baseEnv) { + TableModelUtils.createDataBaseAndTable(baseEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(baseEnv, "test1", "test"); + } + + private void insertData( + String dataBaseName, String tableName, int start, int end, BaseEnv baseEnv) { + TableModelUtils.insertData(dataBaseName, tableName, start, end, baseEnv); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java new file mode 100644 index 0000000000000..068134c3e75e3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; +import static org.awaitility.Awaitility.await; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testAutoDropInHistoricalTransfer() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("mode.snapshot", "true"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQueryCountSql("test"), + "_col0,", + Collections.singleton("100,"), + "test", + handleFailure); + } + + try (final Connection connection = makeItCloseQuietly(senderEnv.getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement()); ) { + ResultSet result = statement.executeQuery("show pipes"); + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(600, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + int pipeNum = 0; + while (result.next()) { + if (!result + .getString(ColumnHeaderConstant.ID) + .contains(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + pipeNum++; + } + } + Assert.assertEquals(0, pipeNum); + } catch (Exception e) { + Assert.fail(); + } + }); + } + } + + @Test + public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("mode.snapshot", "true"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "49"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQueryCountSql("test"), + "_col0,", + Collections.singleton("50,"), + "test", + handleFailure); + } + + try (final Connection connection = makeItCloseQuietly(senderEnv.getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement()); ) { + ResultSet result = statement.executeQuery("show pipes"); + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(600, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + int pipeNum = 0; + while (result.next()) { + if (!result + .getString(ColumnHeaderConstant.ID) + .contains(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + pipeNum++; + } + } + Assert.assertEquals(0, pipeNum); + } catch (Exception e) { + Assert.fail(); + } + }); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java new file mode 100644 index 0000000000000..985bfe752b60e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java @@ -0,0 +1,1048 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionRoleType; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeClusterIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setDataReplicationFactor(2) + .setSchemaReplicationFactor(3) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(3, 3, 180); + receiverEnv.initClusterEnvironment(3, 3, 180); + } + + @Test + public void testMachineDowntimeAsync() { + testMachineDowntime("iotdb-thrift-connector"); + } + + @Test + public void testMachineDowntimeSync() { + testMachineDowntime("iotdb-thrift-sync-connector"); + } + + private void testMachineDowntime(String sink) { + StringBuilder a = new StringBuilder(); + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + a.append(nodeWrapper.getIp()).append(":").append(nodeWrapper.getPort()); + a.append(","); + } + a.deleteCharAt(a.length() - 1); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 1, senderEnv); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", sink); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.node-urls", a.toString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + TableModelUtils.assertCountData("test", "test", 1, receiverEnv); + receiverEnv.getDataNodeWrapper(0).stop(); + + // Ensure that the kill -9 operation is completed + Thread.sleep(5000); + TableModelUtils.insertData("test", "test", 1, 2, senderEnv); + } catch (Exception e) { + fail(e.getMessage()); + } + + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + if (!nodeWrapper.isAlive()) { + continue; + } + TableModelUtils.assertCountData("test", "test", 2, receiverEnv, nodeWrapper); + return; + } + } + + @Test + public void testWithAllParametersInStreamingMode() throws Exception { + testWithAllParameters("true"); + } + + @Test + public void testWithAllParametersInNotStreamingMode() throws Exception { + testWithAllParameters("false"); + } + + private void testWithAllParameters(final String realtimeMode) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "199"); + extractorAttributes.put("mode.streaming", realtimeMode); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "root"); + connectorAttributes.put("connector.password", "root"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 100), + "test", + handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 100, 300, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 200), + "test", + handleFailure); + } + } + + // This function has a certain probability of triggering replica asynchrony. To ensure the success + // of the test, it will be retried 5 times. The exception will be thrown after five retries. + @Test + public void testPipeAfterDataRegionLeaderStop() throws Exception { + for (int retry = 0; retry < 5; retry++) { + try { + if (retry != 0) { + this.setUp(); + } + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "300"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + + final AtomicInteger leaderPort = new AtomicInteger(-1); + final TShowRegionResp showRegionResp = + client.showRegion(new TShowRegionReq().setIsTableModel(true)); + showRegionResp + .getRegionInfoList() + .forEach( + regionInfo -> { + if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { + leaderPort.set(regionInfo.getClientRpcPort()); + } + }); + + int leaderIndex = -1; + for (int i = 0; i < 3; ++i) { + if (senderEnv.getDataNodeWrapper(i).getPort() == leaderPort.get()) { + leaderIndex = i; + try { + senderEnv.shutdownDataNode(i); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + try { + TimeUnit.SECONDS.sleep(1); + } catch (final InterruptedException ignored) { + } + try { + senderEnv.startDataNode(i); + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + } + } + if (leaderIndex == -1) { // ensure the leader is stopped + fail(); + } + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Create a new pipe and write new data + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test1"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test1"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "300"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 301, receiverEnv, handleFailure); + TableModelUtils.assertData("test1", "test1", 0, 301, receiverEnv, handleFailure); + } + return; + } catch (Exception | Error e) { + if (retry < 4) { + this.tearDown(); + } else { + throw e; + } + } + } + } + + @Test + public void testPipeAfterRegisterNewDataNode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + final DataNodeWrapper newDataNode = + senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv, newDataNode); + insertResult = + insertResult + && TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv, newDataNode); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // create a new pipe and write new data + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test1"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test1"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test1", "test1", 0, 400, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + } + } + + @Test + public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test1"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test1"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final Thread t = + new Thread( + () -> { + for (int i = 0; i < 30; ++i) { + try { + client.createPipe( + new TCreatePipeReq("p" + i, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + } catch (final TException e) { + // Not sure if the "createPipe" has succeeded + e.printStackTrace(); + return; + } + try { + Thread.sleep(100); + } catch (final Exception ignored) { + } + } + }); + t.start(); + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + t.join(); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(30, showPipeResult.size()); + } + } + + @Test + public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final AtomicInteger succeedNum = new AtomicInteger(0); + final Thread t = + new Thread( + () -> { + try { + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError( + "test", "test", i, i + 1, senderEnv)) { + succeedNum.incrementAndGet(); + Thread.sleep(100); + } + } + } catch (final InterruptedException ignored) { + } + }); + t.start(); + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + t.join(); + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + TableModelUtils.assertCountData( + "test", "test", succeedNum.get() + 100, receiverEnv, handleFailure); + + try { + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + } catch (final Throwable e) { + e.printStackTrace(); + } + } + } + + @Test + public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + int succeedNum = 0; + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError("test", "test", i, i + 1, senderEnv)) { + succeedNum++; + } + } + + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + TableModelUtils.assertCountData("test", "test", succeedNum + 100, receiverEnv, handleFailure); + + try { + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + } catch (final Throwable e) { + e.printStackTrace(); + } + } + } + + @Test + public void testSenderRestartWhenTransferring() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + int succeedNum = 0; + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError("test", "test", i, i + 1, senderEnv)) { + succeedNum++; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try { + TestUtils.restartCluster(senderEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + TableModelUtils.assertCountData("test", "test", succeedNum + 100, receiverEnv, handleFailure); + } + + @Test + public void testConcurrentlyCreatePipeOfSameName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final AtomicInteger successCount = new AtomicInteger(0); + final List threads = new ArrayList<>(); + for (int i = 0; i < 10; ++i) { + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successCount.incrementAndGet(); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + + for (Thread t : threads) { + t.join(); + } + Assert.assertEquals(1, successCount.get()); + + successCount.set(0); + for (int i = 0; i < 10; ++i) { + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = client.dropPipe("p1"); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successCount.incrementAndGet(); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + for (final Thread t : threads) { + t.join(); + } + + // Assert at least 1 drop operation succeeds + Assert.assertTrue(successCount.get() >= 1); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testCreate10PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(10); + } + + @Test + public void testCreate50PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(50); + } + + @Test + public void testCreate100PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(100); + } + + private void testCreatePipesWithSameConnector(final int pipeCount) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final AtomicInteger successCount = new AtomicInteger(0); + final List threads = new ArrayList<>(); + for (int i = 0; i < pipeCount; ++i) { + final int finalI = i; + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p" + finalI, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + successCount.incrementAndGet(); + } catch (final InterruptedException e) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + for (final Thread t : threads) { + t.join(); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(successCount.get(), showPipeResult.size()); + showPipeResult = + client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + Assert.assertEquals(successCount.get(), showPipeResult.size()); + } + } + + @Test + public void testNegativeTimestamp() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", -100, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertData("test", "test", -100, 100, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", -200, -100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", -200, 100, receiverEnv, handleFailure); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java new file mode 100644 index 0000000000000..0534f046f3a85 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeConnectorCompressionIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + // Override to enable air-gap + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testCompression1() throws Exception { + doTest("iotdb-thrift-connector", "stream", true, "snappy"); + } + + @Test + public void testCompression2() throws Exception { + doTest("iotdb-thrift-connector", "batch", true, "snappy, lzma2"); + } + + @Test + public void testCompression3() throws Exception { + doTest("iotdb-thrift-sync-connector", "stream", false, "snappy, snappy"); + } + + @Test + public void testCompression4() throws Exception { + doTest("iotdb-thrift-sync-connector", "batch", true, "gzip, zstd"); + } + + @Test + public void testCompression5() throws Exception { + doTest("iotdb-air-gap-connector", "stream", false, "lzma2, lz4"); + } + + @Test + public void testCompression6() throws Exception { + doTest("iotdb-air-gap-connector", "batch", true, "lzma2"); + } + + private void doTest( + String connectorType, String realtimeMode, boolean useBatchMode, String compressionTypes) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = + connectorType.contains("air-gap") + ? receiverDataNode.getPipeAirGapReceiverPort() + : receiverDataNode.getPort(); + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", + "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", + "flush"))) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("extractor.realtime.mode", realtimeMode); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", connectorType); + connectorAttributes.put("connector.batch.enable", useBatchMode ? "true" : "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "root"); + connectorAttributes.put("connector.password", "root"); + connectorAttributes.put("connector.compressor", compressionTypes); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,"), + handleFailure); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (3, 3)", + "insert into root.db.d1(time, s1) values (4, 4)", + "insert into root.db.d1(time, s1) values (5, 5)", + "insert into root.db.d1(time, s1) values (6, 6)", + "insert into root.db.d1(time, s1) values (7, 7)", + "insert into root.db.d1(time, s1) values (8, 8)", + "flush"))) { + return; + } + + TableModelUtils.insertData("test", "test", 50, 100, senderEnv, true); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("8,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + } + } + + @Test + public void testZstdCompressorLevel() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (1, 1)", + "insert into root.db.d1(time, s2) values (1, 1)", + "insert into root.db.d1(time, s3) values (1, 1)", + "insert into root.db.d1(time, s4) values (1, 1)", + "insert into root.db.d1(time, s5) values (1, 1)", + "flush"))) { + return; + } + + // Create 5 pipes with different zstd compression levels, p4 and p5 should fail. + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with extractor ('extractor.pattern'='root.db.d1.s1','table-name'='test1','capture.table'='true','capture.tree'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='3'," + + "'connector.rate-limit-bytes-per-second'='2048.0')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p2" + + " with extractor ('extractor.pattern'='root.db.d1.s2','table-name'='test2','capture.table'='true','capture.tree'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='22')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p3" + + " with extractor ('extractor.pattern'='root.db.d1.s3','table-name'='test3','capture.table'='true','capture.tree'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='-131072')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.pattern'='root.db.d1.s4','table-name'='test4','capture.table'='true','capture.tree'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='-131073')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException e) { + // Make sure the error message in IoTDBConnector.java is returned + Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p5" + + " with extractor ('extractor.pattern'='root.db.d1.s5','table-name'='test5','capture.table'='true','capture.tree'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='23')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException e) { + // Make sure the error message in IoTDBConnector.java is returned + Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals( + 3, + showPipeResult.stream() + .filter(info -> !info.id.startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) + .count()); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + TableModelUtils.insertData("test", "test1", 0, 50, senderEnv, true); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + TableModelUtils.insertData("test", "test2", 0, 50, senderEnv, true); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); + TableModelUtils.insertData("test", "test3", 0, 50, senderEnv, true); + + TableModelUtils.assertCountData("test", "test1", 50, receiverEnv, handleFailure); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java new file mode 100644 index 0000000000000..3b82b15d2e439 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeConnectorParallelIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testIoTConnectorParallel() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + final Set expectedResSet = new HashSet<>(); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "true"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.parallel.tasks", "3"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.sg1.d1(time, s1) values (0, 1)", + "insert into root.sg1.d1(time, s1) values (1, 2)", + "insert into root.sg1.d1(time, s1) values (2, 3)", + "insert into root.sg1.d1(time, s1) values (3, 4)", + "flush"))) { + return; + } + TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + expectedResSet.add("0,1.0,"); + expectedResSet.add("1,2.0,"); + expectedResSet.add("2,3.0,"); + expectedResSet.add("3,4.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.sg1.d1.s1,", + expectedResSet, + handleFailure); + + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java new file mode 100644 index 0000000000000..70c7e1605dee5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testDoubleLivingInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.table'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'forwarding-pipe-requests'='true'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + // combination of + // org.apache.iotdb.pipe.it.tablemodel.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving and + // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving + @Test + public void testBasicDoubleLiving() { + boolean insertResult; + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + }; + + // insertion on sender + for (int i = 0; i < 100; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on sender + for (int i = 100; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + for (int i = 200; i < 300; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test", 200, 300, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p2", senderDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on receiver + for (int i = 300; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 300, 400, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 400, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + + // restart cluster + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + // insertion on receiver + for (int i = 400; i < 500; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 400, 500, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + for (int i = 400; i < 500; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 500, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); + } + + @Test + public void testDoubleLivingIsolation() throws Exception { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java new file mode 100644 index 0000000000000..1e02ad9963c29 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeIdempotentIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testCreateTableIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create table test()"); + } + + @Test + public void testCreateTableViewIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.emptyList(), "create view test(s1 field int32) restrict as root.a.**"); + } + + @Test + public void testAlterTableAddColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test()"), "alter table test add column a tag"); + } + + @Test + public void testAlterViewAddColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create view test(s1 field int32) restrict as root.a.**"), + "alter view test add column a tag"); + } + + @Test + public void testAlterTableSetPropertiesIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test()"), + "alter table test set properties ttl=100"); + } + + @Test + public void testAlterViewSetPropertiesIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create view test(s1 field int32) restrict as root.a.**"), + "alter view test set properties ttl=100"); + } + + @Test + public void testAlterTableDropColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test(a tag, b attribute, c int32)"), + "alter table test drop column b"); + } + + @Test + public void testAlterViewDropColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create view test(a tag, s1 field int32) restrict as root.a.**"), + "alter view test drop column a"); + } + + @Test + public void testDropTableIdempotent() throws Exception { + testTableConfigIdempotent(Collections.singletonList("create table test()"), "drop table test"); + } + + @Test + public void testDropViewIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create view test(s1 field int32) restrict as root.a.**"), + "drop view test"); + } + + @Test + public void testRenameViewColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create view test(a tag, s1 field int32) restrict as root.a.**"), + "alter view test rename column a to b"); + } + + @Test + public void testRenameViewIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create view test(s1 field int32) restrict as root.a.**"), + "alter view test rename to test1"); + } + + @Test + public void testTableCreateUserIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create user newUser 'password'"); + } + + @Test + public void testTableDropUserIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create user newUser 'password'"), "drop user newUser"); + } + + @Test + public void testTableCreateRoleIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create role newRole"); + } + + @Test + public void testTableDropRoleIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create role newRole"), "drop role newRole"); + } + + @Test + public void testTableAlterUserIdempotent3() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create user newUser 'password'"), + "alter user newUser set password 'passwd'"); + } + + @Test + public void testTableGrantRoleToUserIdempotent() throws Exception { + testTableConfigIdempotent( + Arrays.asList("create user newUser 'password'", "create role newRole"), + "grant role newRole to newUser"); + } + + @Test + public void testTableRevokeRoleFromUserIdempotent() throws Exception { + testTableConfigIdempotent( + Arrays.asList( + "create user newUser 'password'", + "create role newRole", + "grant role newRole to newUser"), + "revoke role newRole from newUser"); + } + + @Test + public void testTableGrantIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create user newUser 'password'"), + "grant all to user newUser with grant option"); + } + + @Test + public void testTableRevokeIdempotent() throws Exception { + testTableConfigIdempotent( + Arrays.asList( + "create user newUser 'password'", "grant all to user newUser with grant option"), + "revoke all from user newUser"); + } + + @Test + public void testSetTableCommentIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test(a tag)"), "COMMENT ON TABLE test is 'tag'"); + } + + @Test + public void testSetTableColumnCommentIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test(a tag)"), "COMMENT ON COLUMN test.a IS 'tag'"); + } + + private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) + throws Exception { + final String database = "test"; + TableModelUtils.createDatabase(senderEnv, database); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.inclusion.exclusion", ""); + extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, beforeSqlList)) { + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, receiverEnv, testSql)) { + return; + } + + // Create an idempotent conflict + if (!TestUtils.tryExecuteNonQueryWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, testSql)) { + return; + } + + TableModelUtils.createDatabase(senderEnv, "test2"); + + // Assume that the "database" is executed on receiverEnv + TestUtils.assertDataSizeEventuallyOnEnv(receiverEnv, "show databases", 3, null); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java new file mode 100644 index 0000000000000..49e48df4938b7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeMetaIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testTableSync() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "t.*[0-9]"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + final String dbName = "test"; + TableModelUtils.createDatabase(senderEnv, dbName, 300); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + Arrays.asList( + "create table table1(a id, b attribute, c int32) with (ttl=3000)", + "alter table table1 add column d int64", + "alter table table1 drop column c", + "alter table table1 set properties ttl=default", + "insert into table1 (a, b, d) values(1, 1, 1)", + "create table noTransferTable(a id, b attribute, c int32) with (ttl=3000)"))) { + return; + } + + TableModelUtils.createDatabase(senderEnv, "noTransferDatabase", 300); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show tables from test", + "TableName,TTL(ms),", + Collections.singleton("table1,300,"), + dbName); + + // Test devices + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "show devices from table1", "a,b,", Collections.singleton("1,1,"), dbName); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, "insert into table1 (a, b) values(1, 2)")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "show devices from table1", "a,b,", Collections.singleton("1,2,"), dbName); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, "update table1 set b = '3'")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "show devices from table1", "a,b,", Collections.singleton("1,3,"), dbName); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, "delete from table1")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from table1", "a,b,d,", Collections.emptySet(), dbName); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + "delete devices from table1 where a = '1'")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "show devices from table1", "a,b,", Collections.emptySet(), dbName); + + // Will not include no-transfer table + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show tables from test", + "TableName,TTL(ms),", + Collections.singleton("table1,300,"), + dbName); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "desc table1", + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "time,TIMESTAMP,TIME,", + "a,STRING,ID,", + "b,STRING,ATTRIBUTE,", + "d,INT64,MEASUREMENT,")), + dbName); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, "drop table table1")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show tables from test", + "TableName,TTL(ms),", + Collections.emptySet(), + dbName); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, "drop database test")) { + return; + } + + // Will not include no-transfer database + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show databases", + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,"), + (String) null); + } + } + + @Test + public void testNoTree() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.test", + "alter database root.test with schema_region_group_num=2, data_region_group_num=3", + "create timeSeries root.test.d1.s1 int32", + "insert into root.test.d1 (s1) values (1)"))) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show databases", + "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.emptySet()); + } + } + + @Test + public void testNoTable() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.inclusion.exclusion", "data.delete"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.capture.table", "false"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + final String dbName = "test"; + TableModelUtils.createDatabase(senderEnv, dbName, 300); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + Arrays.asList( + "create table table1(a id, b attribute, c int32) with (ttl=3000)", + "alter table table1 add column d int64", + "alter table table1 drop column b", + "alter table table1 set properties ttl=default"))) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show databases", + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,"), + dbName); + } + } + + @Test + public void testAuth() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create user testUser 'password'", "grant all on root.** to user testUser"))) { + return; + } + + final String dbName = "test"; + if (!TestUtils.tryExecuteNonQueriesWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + Arrays.asList( + "grant create on db.tb to user testUser", + "grant drop on database test to user testUser"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "t.*[0-9]"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + "grant alter on any to user testUser with grant option")) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "list privileges of user testUser", + "Role,Scope,Privileges,GrantOption,", + new HashSet<>( + Arrays.asList( + ",,MANAGE_USER,false,", + ",,MANAGE_ROLE,false,", + ",,MAINTAIN,false,", + ",*.*,ALTER,true,", + ",test.*,DROP,false,")), + dbName); + } + } + + @Test + public void testValidation() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe test1 with source ('inclusion'='schema.table') with sink ('ip'='%s', 'port'='%s')", + receiverIp, receiverPort)); + + // Test tree parameters + try { + statement.execute( + String.format( + "create pipe test2 with source ('inclusion'='auth, schema.timeseries') with sink ('ip'='%s', 'port'='%s')", + receiverIp, receiverPort)); + fail(); + } catch (final SQLException e) { + assertEquals("1107: The 'inclusion' string contains illegal path.", e.getMessage()); + } + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe test3 with source ('inclusion'='schema.timeseries') with sink ('ip'='%s', 'port'='%s')", + receiverIp, receiverPort)); + + // Test tree parameters + try { + statement.execute( + String.format( + "create pipe test4 with source ('inclusion'='auth, schema.table') with sink ('ip'='%s', 'port'='%s')", + receiverIp, receiverPort)); + fail(); + } catch (final SQLException e) { + assertEquals("1107: The 'inclusion' string contains illegal path.", e.getMessage()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java new file mode 100644 index 0000000000000..9d635cd809528 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeNullValueIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + private enum InsertType { + SESSION_INSERT_RECORD, + SESSION_INSERT_TABLET, + SQL_INSERT, + } + + private void testInsertNullValueTemplate( + final InsertType insertType, final boolean withParsing, final String realtime) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + + if (insertType == InsertType.SESSION_INSERT_TABLET) { + TableModelUtils.insertTablet("test", "test", 0, 200, senderEnv, true); + } else if (insertType == InsertType.SQL_INSERT) { + TableModelUtils.insertData("test", "test", 0, 200, senderEnv, true); + } + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("realtime-mode", realtime); + extractorAttributes.put("user", "root"); + if (withParsing) { + extractorAttributes.put("start-time", "150"); + extractorAttributes.put("end-time", "249"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test"); + } + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("test", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + if (insertType == InsertType.SESSION_INSERT_TABLET) { + TableModelUtils.insertTablet("test", "test", 200, 400, senderEnv, true); + } else if (insertType == InsertType.SQL_INSERT) { + TableModelUtils.insertData("test", "test", 200, 400, senderEnv, true); + } + + if (withParsing) { + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + return; + } + TableModelUtils.assertCountData("test", "test", 400, receiverEnv, handleFailure); + } + + // ---------------------- // + // Scenario 1: SQL Insert // + // ---------------------- // + @Test + public void testSQLInsertWithParsingForcedLog() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "forced-log"); + } + + @Test + public void testSQLInsertWithoutParsingForcedLog() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "forced-log"); + } + + @Test + public void testSQLInsertWithParsingFile() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "file"); + } + + @Test + public void testSQLInsertWithoutParsingFile() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "file"); + } + + @Test + public void testSQLInsertWithParsingStream() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "stream"); + } + + @Test + public void testSQLInsertWithoutParsingStream() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "stream"); + } + + // --------------------------------- // + // Scenario 2: Session Insert Tablet // + // --------------------------------- // + @Test + public void testSessionInsertTabletWithParsingForcedLog() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "forced-log"); + } + + @Test + public void testSessionInsertTabletWithoutParsingForcedLog() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "forced-log"); + } + + @Test + public void testSessionInsertTabletWithParsingFile() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "file"); + } + + @Test + public void testSessionInsertTabletWithoutParsingFile() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "file"); + } + + @Test + public void testSessionInsertTabletWithParsingStream() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "stream"); + } + + @Test + public void testSessionInsertTabletWithoutParsingStream() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "stream"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java new file mode 100644 index 0000000000000..c7a0efe370a45 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java @@ -0,0 +1,547 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + private static final int generateDataSize = 1000; + + @Test + public void insertTablet() { + prepareTypeConversionTest( + (ITableSession senderSession, ITableSession receiverSession, Tablet tablet) -> { + senderSession.insert(tablet); + }, + false); + } + + @Test + public void insertTabletReceiveByTsFile() { + prepareTypeConversionTest( + (ITableSession senderSession, ITableSession receiverSession, Tablet tablet) -> { + senderSession.insert(tablet); + }, + true); + } + + private SessionDataSet query( + ITableSession session, List measurementSchemas, String tableName) + throws IoTDBConnectionException, StatementExecutionException { + String sql = "select "; + StringBuilder param = new StringBuilder(); + for (IMeasurementSchema schema : measurementSchemas) { + param.append(schema.getMeasurementName()); + param.append(','); + } + + sql = sql + param + "time from " + tableName + " ORDER BY time ASC"; + session.executeNonQueryStatement("use test"); + return session.executeQueryStatement(sql); + } + + private void prepareTypeConversionTest( + CheckedTriConsumer executeDataWriteOperation, + boolean isTsFile) { + List> measurementSchemas = + generateMeasurementSchemas(); + Tablet tablet = generateTabletAndMeasurementSchema(measurementSchemas, "test"); + createDatabaseAndTable(measurementSchemas, true, tablet.getColumnTypes(), senderEnv); + createDatabaseAndTable(measurementSchemas, false, tablet.getColumnTypes(), receiverEnv); + try (ITableSession senderSession = senderEnv.getTableSessionConnection(); + ITableSession receiverSession = receiverEnv.getTableSessionConnection()) { + senderSession.executeNonQueryStatement("use test"); + receiverSession.executeNonQueryStatement("use test"); + if (isTsFile) { + // Send TsFile data to receiver + executeDataWriteOperation.accept(senderSession, receiverSession, tablet); + senderSession.executeNonQueryStatement("flush"); + createDataPipe(true); + } else { + // Send Tablet data to receiver + createDataPipe(false); + // The actual implementation logic of inserting data + executeDataWriteOperation.accept(senderSession, receiverSession, tablet); + senderSession.executeNonQueryStatement("flush"); + } + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + // Verify receiver data + long timeoutSeconds = 600; + List> expectedValues = + generateTabletResultSetForTable(tablet, measurementSchemas); + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + validateResultSet( + query(receiverSession, tablet.getSchemas(), tablet.getTableName()), + expectedValues, + tablet.getTimestamps()); + } catch (Exception | Error e) { + handleFailure.accept(e.getMessage()); + fail(e.getMessage()); + } + }); + } catch (Exception e) { + fail(e.getMessage()); + } + tablet.reset(); + } + + private void createDatabaseAndTable( + List> measurementSchemas, + boolean isLeft, + List categories, + BaseEnv env) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < measurementSchemas.size(); i++) { + final MeasurementSchema measurement = + isLeft ? measurementSchemas.get(i).getLeft() : measurementSchemas.get(i).getRight(); + builder.append( + String.format( + "%s %s %s,", + measurement.getMeasurementName(), measurement.getType(), categories.get(i).name())); + } + builder.deleteCharAt(builder.length() - 1); + String tableCreation = + String.format("create table if not exists test (%s)", builder.toString()); + TestUtils.tryExecuteNonQueriesWithRetry( + null, + "table", + env, + Arrays.asList("create database if not exists test", "use test", tableCreation)); + } + + private void createDataPipe(boolean isTSFile) { + String sql = + String.format( + "create pipe test" + + " with source ('source'='iotdb-source','realtime.mode'='%s')" + + " with processor ('processor'='do-nothing-processor')" + + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", + isTSFile ? "file" : "forced-log", + receiverEnv.getIP(), + receiverEnv.getPort(), + isTSFile ? "tsfile" : "tablet"); + TestUtils.tryExecuteNonQueriesWithRetry( + null, BaseEnv.TABLE_SQL_DIALECT, senderEnv, Collections.singletonList(sql)); + } + + private void validateResultSet( + SessionDataSet dataSet, List> values, long[] timestamps) + throws IoTDBConnectionException, StatementExecutionException { + int index = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + List fields = record.getFields(); + List rowValues = values.get(index++); + for (int i = 0; i < fields.size(); i++) { + Field field = fields.get(i); + if (field.getDataType() == null) { + assertNull(rowValues.get(i)); + continue; + } + switch (field.getDataType()) { + case INT64: + case TIMESTAMP: + assertEquals(field.getLongV(), (long) rowValues.get(i)); + break; + case DATE: + assertEquals(field.getDateV(), rowValues.get(i)); + break; + case BLOB: + case TEXT: + case STRING: + assertEquals(field.getBinaryV(), rowValues.get(i)); + break; + case INT32: + assertEquals(field.getIntV(), (int) rowValues.get(i)); + break; + case DOUBLE: + assertEquals(0, Double.compare(field.getDoubleV(), (double) rowValues.get(i))); + break; + case FLOAT: + assertEquals(0, Float.compare(field.getFloatV(), (float) rowValues.get(i))); + break; + } + } + } + assertEquals(values.size(), index); + } + + private void createTestDataForBoolean(Tablet tablet, int j) { + Random random = new Random(); + for (int i = 0; i < generateDataSize; i++) { + if (random.nextBoolean()) { + tablet.addValue(i, j, random.nextBoolean()); + } + } + } + + private void createTestDataForInt32(Tablet tablet, int j) { + Random random = new Random(); + for (int i = 0; i < generateDataSize; i++) { + if (random.nextBoolean()) { + tablet.addValue(i, j, random.nextInt()); + } + } + } + + private void createTestDataForInt64(Tablet tablet, int j) { + Random random = new Random(); + for (int i = 0; i < generateDataSize; i++) { + if (random.nextBoolean()) { + tablet.addValue(i, j, random.nextLong()); + } + } + } + + private void createTestDataForFloat(Tablet tablet, int j) { + Random random = new Random(); + for (int i = 0; i < generateDataSize; i++) { + if (random.nextBoolean()) { + tablet.addValue(i, j, random.nextFloat()); + } + } + } + + private void createTestDataForDouble(Tablet tablet, int j) { + Random random = new Random(); + for (int i = 0; i < generateDataSize; i++) { + if (random.nextBoolean()) { + tablet.addValue(i, j, random.nextDouble()); + } + } + } + + private void createTestDataForTimestamp(Tablet tablet, int j) { + Random random = new Random(); + long time = new Date().getTime(); + for (int i = 0; i < generateDataSize; i++) { + if (random.nextBoolean()) { + tablet.addValue(i, j, time); + } + } + } + + private void createTestDataForTimeColumn(Tablet tablet) { + long time = new Date().getTime(); + for (int i = 0; i < generateDataSize; i++) { + tablet.addTimestamp(i, time++); + } + } + + private void createTestDataForDate(Tablet tablet, int j) { + int year = 2025; + int month = 1; + int day = 1; + for (int i = 0; i < generateDataSize; i++) { + tablet.addValue(i, j, DateUtils.parseIntToLocalDate(year * 10000 + (month * 100) + day)); + // update + day++; + if (day > 28) { + day = 1; + month++; + if (month > 12) { + month = 1; + year++; + } + } + } + } + + private void createTestDataForString(Tablet tablet, int j) { + String[] stringData = { + "Hello", + "Hello World!", + "This is a test.", + "IoTDB Hello World!!!!", + "IoTDB is an excellent time series database!!!!!!!!!", + "12345678910!!!!!!!!", + "123456", + "1234567.123213", + "21232131.21", + "enable = true", + "true", + "false", + }; + for (int i = 0; i < generateDataSize; i++) { + tablet.addValue( + i, j, stringData[(i % stringData.length)].getBytes(TSFileConfig.STRING_CHARSET)); + } + } + + private List> generateTabletResultSetForTable( + final Tablet tablet, List> pairs) { + List> insertRecords = new ArrayList<>(tablet.getRowSize()); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + for (int i = 0; i < tablet.getRowSize(); i++) { + List insertRecord = new ArrayList<>(); + for (int j = 0; j < schemas.size(); j++) { + TSDataType sourceType = pairs.get(j).left.getType(); + TSDataType targetType = pairs.get(j).right.getType(); + Object value = null; + switch (sourceType) { + case INT64: + case TIMESTAMP: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((long[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case INT32: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((int[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case DOUBLE: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((double[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case FLOAT: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((float[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case DATE: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) + ? null + : DateUtils.parseDateExpressionToInt(((LocalDate[]) values[j])[i])); + insertRecord.add(convert(value, targetType)); + break; + case TEXT: + case STRING: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((Binary[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case BLOB: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((Binary[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case BOOLEAN: + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.getBitMaps()[j].isMarked(i) ? null : ((boolean[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + } + } + insertRecord.add(tablet.getTimestamp(i)); + insertRecords.add(insertRecord); + } + + return insertRecords; + } + + private Object convert(Object value, TSDataType targetType) { + if (value == null) { + return null; + } + switch (targetType) { + case DATE: + return DateUtils.parseIntToLocalDate((Integer) value); + case TEXT: + case STRING: + return value; + } + return value; + } + + private Tablet generateTabletAndMeasurementSchema( + List> pairs, String tableName) { + List schemaList = new ArrayList<>(); + for (Pair pair : pairs) { + schemaList.add(pair.left); + } + + final List columnTypes = generateTabletColumnCategory(pairs.size()); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + generateDataSize); + tablet.initBitMaps(); + createTestDataForTimeColumn(tablet); + for (int i = 0; i < pairs.size(); i++) { + MeasurementSchema schema = pairs.get(i).left; + switch (schema.getType()) { + case INT64: + createTestDataForInt64(tablet, i); + break; + case INT32: + createTestDataForInt32(tablet, i); + break; + case TIMESTAMP: + createTestDataForTimestamp(tablet, i); + break; + case DOUBLE: + createTestDataForDouble(tablet, i); + break; + case FLOAT: + createTestDataForFloat(tablet, i); + break; + case DATE: + createTestDataForDate(tablet, i); + break; + case STRING: + case BLOB: + case TEXT: + createTestDataForString(tablet, i); + break; + case BOOLEAN: + createTestDataForBoolean(tablet, i); + break; + } + } + + return tablet; + } + + private List generateTabletColumnCategory(int size) { + List columnTypes = new ArrayList<>(size); + columnTypes.add(ColumnCategory.TAG); + columnTypes.add(ColumnCategory.TAG); + columnTypes.add(ColumnCategory.TAG); + columnTypes.add(ColumnCategory.TAG); + for (int i = 0; i < size - 4; i++) { + columnTypes.add(ColumnCategory.FIELD); + } + return columnTypes; + } + + private List> generateMeasurementSchemas() { + TSDataType[] dataTypes = { + TSDataType.STRING, + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.TIMESTAMP, + TSDataType.BOOLEAN, + TSDataType.DATE, + TSDataType.DOUBLE, + TSDataType.FLOAT, + TSDataType.INT32, + TSDataType.INT64 + }; + List> pairs = new ArrayList<>(); + + for (int t = 0; t < 4; t++) { + pairs.add( + new Pair<>( + new MeasurementSchema("s" + t, TSDataType.STRING), + new MeasurementSchema("s" + t, TSDataType.STRING))); + } + + for (TSDataType type : dataTypes) { + for (TSDataType dataType : dataTypes) { + String id = String.format("%s2%s", type.name(), dataType.name()); + pairs.add(new Pair<>(new MeasurementSchema(id, type), new MeasurementSchema(id, dataType))); + } + } + return pairs; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java new file mode 100644 index 0000000000000..bfc47abb1584b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java @@ -0,0 +1,647 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.RpcUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.Pair; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.nio.charset.StandardCharsets; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.function.Consumer; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeTypeConversionIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + private static final int generateDataSize = 100; + + // Test for converting BOOLEAN to OtherType + @Test + public void testBooleanToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.DATE); + } + + // Test for converting INT32 to OtherType + @Test + public void testInt32ToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.DATE); + } + + // Test for converting INT64 to OtherType + @Test + public void testInt64ToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.DATE); + } + + // Test for converting FLOAT to OtherType + @Test + public void testFloatToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.DATE); + } + + // Test for converting DOUBLE to OtherType + @Test + public void testDoubleToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.DATE); + } + + // Test for converting TEXT to OtherType + @Test + public void testTextToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.DATE); + } + + // Test for converting TIMESTAMP to OtherType + @Test + public void testTimestampToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.DATE); + } + + // Test for converting DATE to OtherType + @Test + public void testDateToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.TIMESTAMP); + } + + // Test for converting BLOB to OtherType + @Test + public void testBlobToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.DATE); + } + + // Test for converting STRING to OtherType + @Test + public void testStringToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.TIMESTAMP); + } + + private void executeAndVerifyTypeConversion(TSDataType source, TSDataType target) { + List pairs = prepareTypeConversionTest(source, target); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + String.format("select time,status,s1 from %s2%s", source.name(), target.name()), + "time,status,s1,", + createExpectedResultSet(pairs, source, target), + "test", + handleFailure); + } + + private List prepareTypeConversionTest(TSDataType sourceType, TSDataType targetType) { + String sourceTypeName = sourceType.name(); + String targetTypeName = targetType.name(); + + createDataBaseAndTable(sourceTypeName, targetTypeName, sourceTypeName, senderEnv); + createDataBaseAndTable(sourceTypeName, targetTypeName, targetTypeName, receiverEnv); + + List pairs = createTestDataForType(sourceTypeName); + + executeDataInsertions(pairs, sourceType, targetType); + return pairs; + } + + private void createDataBaseAndTable( + String sourceTypeName, String targetTypeName, String dataType, BaseEnv env) { + String timeSeriesCreationQuery = + String.format( + "create table %s2%s (s1 string tag, status %s field)", + sourceTypeName, targetTypeName, dataType); + TestUtils.tryExecuteNonQueriesWithRetry( + null, + BaseEnv.TABLE_SQL_DIALECT, + env, + Arrays.asList("create database if not exists test", "use test", timeSeriesCreationQuery)); + } + + private void createDataPipe() { + String sql = + String.format( + "create pipe test" + + " with source ('source'='iotdb-source','realtime.mode'='forced-log','realtime.enable'='true','history.enable'='false')" + + " with processor ('processor'='do-nothing-processor')" + + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='tablet')", + receiverEnv.getIP(), receiverEnv.getPort()); + TestUtils.tryExecuteNonQueriesWithRetry( + null, BaseEnv.TABLE_SQL_DIALECT, senderEnv, Collections.singletonList(sql)); + } + + private List createTestDataForType(String sourceType) { + switch (sourceType) { + case "BOOLEAN": + return createTestDataForBoolean(); + case "INT32": + return createTestDataForInt32(); + case "INT64": + return createTestDataForInt64(); + case "FLOAT": + return createTestDataForFloat(); + case "DOUBLE": + return createTestDataForDouble(); + case "TEXT": + return createTestDataForText(); + case "TIMESTAMP": + return createTestDataForTimestamp(); + case "DATE": + return createTestDataForDate(); + case "BLOB": + return createTestDataForBlob(); + case "STRING": + return createTestDataForString(); + default: + throw new UnsupportedOperationException("Unsupported data type: " + sourceType); + } + } + + private void executeDataInsertions( + List testData, TSDataType sourceType, TSDataType targetType) { + switch (sourceType) { + case STRING: + case TEXT: + TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + createInsertStatementsForString(testData, sourceType.name(), targetType.name())); + return; + case TIMESTAMP: + TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + createInsertStatementsForTimestamp(testData, sourceType.name(), targetType.name())); + return; + case DATE: + TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + createInsertStatementsForLocalDate(testData, sourceType.name(), targetType.name())); + return; + case BLOB: + TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + createInsertStatementsForBlob(testData, sourceType.name(), targetType.name())); + return; + default: + TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + createInsertStatementsForNumeric(testData, sourceType.name(), targetType.name())); + } + } + + private List createInsertStatementsForString( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into %s2%s(time,status,s1) values (%s,'%s','t1')", + sourceType, + targetType, + pair.left, + new String(((Binary) (pair.right)).getValues(), StandardCharsets.UTF_8))); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForNumeric( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into %s2%s (time,status,s1) values (%s,%s,'t1')", + sourceType, targetType, pair.left, pair.right)); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForTimestamp( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into %s2%s(time,status,s1) values (%s,%s,'t1')", + sourceType, targetType, pair.left, pair.right)); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForLocalDate( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into %s2%s(time,status,s1) values (%s,'%s','t1')", + sourceType, targetType, pair.left, DateUtils.formatDate((Integer) pair.right))); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForBlob( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + String value = BytesUtils.parseBlobByteArrayToString(((Binary) pair.right).getValues()); + executes.add( + String.format( + "insert into %s2%s(time,status,s1) values (%s,X'%s','t1')", + sourceType, targetType, pair.left, value.substring(2))); + } + executes.add("flush"); + return executes; + } + + private Set createExpectedResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + switch (targetType) { + case TIMESTAMP: + return generateTimestampResultSet(pairs, sourceType, targetType); + case DATE: + return generateLocalDateResultSet(pairs, sourceType, targetType); + case BLOB: + return generateBlobResultSet(pairs, sourceType, targetType); + case TEXT: + case STRING: + return generateStringResultSet(pairs, sourceType, targetType); + default: + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,t1,", + toTimestamp((long) pair.left), + ValueConverter.convert(sourceType, targetType, pair.right))); + } + return resultSet; + } + } + + private Set generateTimestampResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,t1,", + toTimestamp((long) pair.left), + toTimestamp((long) ValueConverter.convert(sourceType, targetType, pair.right)))); + } + return resultSet; + } + + private Set generateLocalDateResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,t1,", + toTimestamp((long) pair.left), + DateUtils.formatDate( + (Integer) ValueConverter.convert(sourceType, targetType, pair.right)))); + } + return resultSet; + } + + private Set generateBlobResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,t1,", + toTimestamp((long) pair.left), + BytesUtils.parseBlobByteArrayToString( + ((Binary) ValueConverter.convert(sourceType, targetType, pair.right)) + .getValues()))); + } + return resultSet; + } + + private Set generateStringResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,t1,", + toTimestamp((long) pair.left), + new String( + ((Binary) ValueConverter.convert(sourceType, targetType, pair.right)).getValues(), + StandardCharsets.UTF_8))); + } + return resultSet; + } + + private List createTestDataForBoolean() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextBoolean())); + } + return pairs; + } + + private List createTestDataForInt32() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextInt())); + } + pairs.add(new Pair<>(generateDataSize + 1L, -1)); + pairs.add(new Pair<>(generateDataSize + 2L, -2)); + pairs.add(new Pair<>(generateDataSize + 3L, -3)); + return pairs; + } + + private List createTestDataForInt64() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextLong())); + } + pairs.add(new Pair<>(generateDataSize + 1L, -1L)); + pairs.add(new Pair<>(generateDataSize + 2L, -2L)); + pairs.add(new Pair<>(generateDataSize + 3L, -3L)); + return pairs; + } + + private List createTestDataForFloat() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextFloat())); + } + return pairs; + } + + private List createTestDataForDouble() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextDouble())); + } + return pairs; + } + + private List createTestDataForText() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Binary((String.valueOf(i)).getBytes(StandardCharsets.UTF_8)))); + } + pairs.add( + new Pair(generateDataSize + 1L, new Binary("Hello".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 2L, new Binary("Hello World!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 3L, new Binary("This is a test.".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 4L, + new Binary("IoTDB Hello World!!!!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 5L, + new Binary( + "IoTDB is an excellent time series database!!!!!!!!!" + .getBytes(StandardCharsets.UTF_8)))); + return pairs; + } + + private List createTestDataForTimestamp() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Date().getTime() + i)); + } + return pairs; + } + + private List createTestDataForDate() { + List pairs = new ArrayList<>(); + int year = 2023; + int month = 1; + int day = 1; + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, year * 10000 + (month * 100) + day)); + + // update + day++; + if (day > 28) { + day = 1; + month++; + if (month > 12) { + month = 1; + year++; + } + } + } + return pairs; + } + + private List createTestDataForBlob() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Binary((String.valueOf(i)).getBytes(StandardCharsets.UTF_8)))); + } + pairs.add( + new Pair(generateDataSize + 1L, new Binary("Hello".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 2L, new Binary("Hello World!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 3L, new Binary("This is a test.".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 4L, + new Binary("IoTDB Hello World!!!!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 5L, + new Binary( + "IoTDB is an excellent time series database!!!!!!!!!" + .getBytes(StandardCharsets.UTF_8)))); + return pairs; + } + + private List createTestDataForString() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Binary((String.valueOf(i)).getBytes(StandardCharsets.UTF_8)))); + } + pairs.add( + new Pair(generateDataSize + 1L, new Binary("Hello".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 2L, new Binary("Hello World!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 3L, new Binary("This is a test.".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 4L, + new Binary("IoTDB Hello World!!!!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 5L, + new Binary( + "IoTDB is an excellent time series database!!!!!!!!!" + .getBytes(StandardCharsets.UTF_8)))); + return pairs; + } + + private String toTimestamp(long timestamp) { + return RpcUtils.formatDatetime("default", "ms", timestamp, ZoneOffset.UTC); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java new file mode 100644 index 0000000000000..4aa06e7acf314 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto; + +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractPipeDualTreeModelAutoIT { + + protected BaseEnv senderEnv; + protected BaseEnv receiverEnv; + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + setupConfig(); + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + protected void setupConfig() { + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + } + + @After + public final void tearDown() { + senderEnv.cleanClusterEnvironment(); + receiverEnv.cleanClusterEnvironment(); + } + + protected static void awaitUntilFlush(BaseEnv env) { + Awaitility.await() + .atMost(1, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + if (!TestUtils.tryExecuteNonQueryWithRetry(env, "flush")) { + return false; + } + return env.getDataNodeWrapperList().stream() + .anyMatch( + wrapper -> { + int fileNum = 0; + File sequence = new File(buildDataPath(wrapper, true)); + File unsequence = new File(buildDataPath(wrapper, false)); + if (sequence.exists() && sequence.listFiles() != null) { + fileNum += Objects.requireNonNull(sequence.listFiles()).length; + } + if (unsequence.exists() && unsequence.listFiles() != null) { + fileNum += Objects.requireNonNull(unsequence.listFiles()).length; + } + return fileNum > 0; + }); + }); + } + + private static String buildDataPath(DataNodeWrapper wrapper, boolean isSequence) { + String nodePath = wrapper.getNodePath(); + return nodePath + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + "datanode" + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + (isSequence ? IoTDBConstant.SEQUENCE_FOLDER_NAME : IoTDBConstant.UNSEQUENCE_FOLDER_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java new file mode 100644 index 0000000000000..564bc34aee532 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java @@ -0,0 +1,553 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeAlterIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testBasicAlterPipe() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create pipe + final String sql = + String.format( + "create pipe a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + long lastCreationTime; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Record last creation time + lastCreationTime = showPipeResult.get(0).creationTime; + } + + // Stop pipe + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("stop pipe a2b"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + } + + // Alter pipe (modify) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('source.pattern'='root.test2')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("__system.sql-dialect=tree")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b replace source ('source'='iotdb-source', 'source.path'='root.test1.**')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); + Assert.assertFalse( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('sink.batch.enable'='false')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Start pipe + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("start pipe a2b"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Alter pipe (replace) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b replace processor ('processor'='tumbling-time-sampling-processor')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('connector.batch.enable'='true')"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b replace source ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b replace processor ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertFalse( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ()"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertFalse( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + } + + @Test + public void testAlterPipeFailure() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // alter non-existed pipe + String sql = + String.format( + "alter pipe a2b modify sink ('node-urls'='%s', 'batch.enable'='true')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + fail(); + } catch (SQLException ignore) { + } + + // Create pipe + sql = + String.format( + "create pipe a2b with sink ('node-urls'='%s', 'batch.enable'='false')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testAlterPipeSourceAndProcessor() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create pipe + final String sql = + String.format( + "create pipe a2b with source ('source' = 'iotdb-source','source.path' = 'root.db.d1.**') with processor ('processor'='tumbling-time-sampling-processor', 'processor.tumbling-time.interval-seconds'='1', 'processor.down-sampling.split-file'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", + receiverDataNode.getIpAndPortString()); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Insert data on sender + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (1000, 1), (1500, 2), (2000, 3), (2500, 4), (3000, 5)", + "flush"))) { + fail(); + } + + // Check data on receiver + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1000,1.0,"); + expectedResSet.add("2000,3.0,"); + expectedResSet.add("3000,5.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.db.**", "Time,root.db.d1.at1,", expectedResSet); + + // Alter pipe (modify 'source.path', 'source.inclusion' and + // 'processor.tumbling-time.interval-seconds') + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**', 'source.inclusion'='all') modify processor ('processor.tumbling-time.interval-seconds'='2')"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Insert data on sender + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d2 (time, at1) values (11000, 1), (11500, 2), (12000, 3), (12500, 4), (13000, 5)", + "flush"))) { + fail(); + } + + // Insert data on sender + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (11000, 1), (11500, 2), (12000, 3), (12500, 4), (13000, 5)", + "flush"))) { + fail(); + } + + // Check data on receiver + expectedResSet.clear(); + expectedResSet.add("11000,null,1.0,"); + expectedResSet.add("13000,null,5.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db.** where time > 10000", + "Time,root.db.d1.at1,root.db.d2.at1,", + expectedResSet); + + // Create database on sender + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "create timeSeries root.db.d2.at2 int32")) { + fail(); + } + + // Check database on receiver + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeSeries", "count(timeseries),", Collections.singleton("3,")); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java new file mode 100644 index 0000000000000..45169c416300a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeConnectorParallelIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testIoTConnectorParallel() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Set expectedResSet = new HashSet<>(); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.parallel.tasks", "3"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.sg1.d1(time, s1) values (0, 1)", + "insert into root.sg1.d1(time, s1) values (1, 2)", + "insert into root.sg1.d1(time, s1) values (2, 3)", + "insert into root.sg1.d1(time, s1) values (3, 4)", + "flush"))) { + return; + } + + expectedResSet.add("0,1.0,"); + expectedResSet.add("1,2.0,"); + expectedResSet.add("2,3.0,"); + expectedResSet.add("3,4.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.sg1.d1.s1,", expectedResSet); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java new file mode 100644 index 0000000000000..1dd102f0e5961 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeDataSinkIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "log"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.realtime-first", "false"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,")))); + } + } + + @Test + public void testSinkTabletFormat() throws Exception { + testSinkFormat("tablet"); + } + + @Test + public void testSinkTsFileFormat() throws Exception { + testSinkFormat("tsfile"); + } + + private void testSinkFormat(final String format) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "forced-log"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.format", format); + connectorAttributes.put("connector.realtime-first", "false"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,")))); + } + } + + @Test + public void testLegacyConnector() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.realtime.mode", "log"); + + connectorAttributes.put("sink", "iotdb-legacy-pipe-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + + // This version does not matter since it's no longer checked by the legacy receiver + connectorAttributes.put("sink.version", "1.3"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.singleton("0,1.0,")); + } + } + + @Test + public void testReceiverAutoCreateByLog() throws Exception { + testReceiverAutoCreate( + new HashMap() { + { + put("source.realtime.mode", "forced-log"); + } + }); + } + + @Test + public void testReceiverAutoCreateByFile() throws Exception { + testReceiverAutoCreate( + new HashMap() { + { + put("source.realtime.mode", "batch"); + } + }); + } + + @Test + public void testReceiverAutoCreateWithPattern() throws Exception { + testReceiverAutoCreate( + new HashMap() { + { + put("source.realtime.mode", "batch"); + put("source.path", "root.ln.wf01.wt0*.*"); + } + }); + } + + private void testReceiverAutoCreate(final Map extractorAttributes) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create timeSeries root.ln.wf01.wt01.boolean boolean", + "create timeSeries root.ln.wf01.wt01.int32 int32", + "create timeSeries root.ln.wf01.wt01.int64 int64", + "create timeSeries root.ln.wf01.wt01.float float", + "create timeSeries root.ln.wf01.wt01.double double", + "create timeSeries root.ln.wf01.wt01.time_stamp timestamp", + "create timeSeries root.ln.wf01.wt01.date date", + "create timeSeries root.ln.wf01.wt01.text text", + "create timeSeries root.ln.wf01.wt01.string string", + "create timeSeries root.ln.wf01.wt01.blob blob", + "create aligned timeSeries root.ln.wf01.wt02(int32 int32, boolean boolean)", + "insert into root.ln.wf01.wt01(time, boolean, int32, int64, float, double, time_stamp, date, text, string, blob) values (20000, false, 123, 321, 13.3, 14.4, now(), '2000-12-13', 'abc', 'def', X'f103')", + "insert into root.ln.wf01.wt02(time, int32, boolean) values (20000, 123, false)", + // For pattern parse + "insert into root.ln.wf01.wt11(time, redundant_data) values (20000, -1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeSeries root.ln.wf01.wt01.*", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + "root.ln.wf01.wt01.boolean,null,root.ln,BOOLEAN,RLE,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.int32,null,root.ln,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.int64,null,root.ln,INT64,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.float,null,root.ln,FLOAT,GORILLA,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.double,null,root.ln,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.time_stamp,null,root.ln,TIMESTAMP,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.date,null,root.ln,DATE,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.text,null,root.ln,TEXT,PLAIN,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.string,null,root.ln,STRING,PLAIN,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.blob,null,root.ln,BLOB,PLAIN,LZ4,null,null,null,null,BASE,")))); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show devices root.ln.wf01.wt02", + "Device,IsAligned,Template,TTL(ms),", + Collections.singleton("root.ln.wf01.wt02,true,null,INF,")); + } + } + + @Test + public void testSyncLoadTsFile() throws Exception { + testReceiverLoadTsFile("sync"); + } + + @Test + public void testAsyncLoadTsFile() throws Exception { + testReceiverLoadTsFile("async"); + } + + private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "forced-log"); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + connectorAttributes.put("sink.load-tsfile-strategy", loadTsFileStrategy); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,")))); + } + } + + @Test + public void testSyncLoadTsFileWithoutVerify() throws Exception { + testLoadTsFileWithoutVerify("sync"); + } + + @Test + public void testAsyncLoadTsFileWithoutVerify() throws Exception { + testLoadTsFileWithoutVerify("async"); + } + + private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "batch"); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + connectorAttributes.put("sink.load-tsfile-strategy", loadTsFileStrategy); + connectorAttributes.put("sink.tsfile.validation", "false"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create timeSeries root.vehicle.d0.s1 int32", + "insert into root.vehicle.d0(time, s1) values (2, 1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java new file mode 100644 index 0000000000000..32075c4040cc2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java @@ -0,0 +1,1052 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeExtractorIT extends AbstractPipeDualTreeModelAutoIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Disable sender compaction for tsfile determination in loose range test + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testExtractorValidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + // ---------------------------------------------------------- // + // Scenario 1: when 'extractor.history.enable' is set to true // + // ---------------------------------------------------------- // + + // Scenario 1.1: test when 'extractor.history.start-time' and 'extractor.history.end-time' are + // not set + final String p1_1 = + String.format( + "create pipe p1_1" + + " with extractor (" + + "'extractor.history.enable'='true')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p1_1); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 1.2: test when only 'extractor.history.start-time' is set + final String p1_2 = + String.format( + "create pipe p1_2" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.history.start-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p1_2); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 1.3: test when only 'extractor.history.end-time' is set + final String p1_3 = + String.format( + "create pipe p1_3" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.history.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p1_3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 1.4: test when 'extractor.history.start-time' equals 'extractor.history.end-time' + final String p1_4 = + String.format( + "create pipe p1_4" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.history.start-time'='2000.01.01T08:00:00'," + + "'extractor.history.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p1_4); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 1.5: test when 'extractor.history.end-time' is future time + final String p1_5 = + String.format( + "create pipe p1_5" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.history.start-time'='2000.01.01T08:00:00'," + + "'extractor.history.end-time'='2100.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p1_5); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + assertPipeCount(5); + + // ---------------------------------------------------------------------- // + // Scenario 2: when 'source.start-time' or 'source.end-time' is specified // + // ---------------------------------------------------------------------- // + + // Scenario 2.1: test when only 'source.start-time' is set + final String p2_1 = + String.format( + "create pipe p2_1" + + " with source (" + + "'source.start-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p2_1); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 2.2: test when only 'source.end-time' is set + final String p2_2 = + String.format( + "create pipe p2_2" + + " with source (" + + "'source.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p2_2); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 2.3: test when 'source.start-time' equals 'source.end-time' + final String p2_3 = + String.format( + "create pipe p2_3" + + " with source (" + + "'source.start-time'='2000.01.01T08:00:00'," + + "'source.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p2_3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Scenario 2.4: test when only when 'source.start-time' is less than 'source.end-time' + final String p2_4 = + String.format( + "create pipe p2_4" + + " with source (" + + "'source.start-time'='2000.01.01T08:00:00'," + + "'source.end-time'='2100.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p2_4); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + assertPipeCount(9); + + // ---------------------------------------------------------------------- // + // Scenario 3: when 'source.start-time' or 'source.end-time' is timestamp // + // ---------------------------------------------------------------------- // + + // Scenario 3.1: test when 'source.start-time' is timestamp string + final String p3_1 = + String.format( + "create pipe p3_1" + + " with source (" + + "'source.start-time'='1000'," + + "'source.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p3_1); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + assertPipeCount(10); + } + + @Test + public void testExtractorInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + // Scenario 1: invalid 'extractor.history.start-time' + final String formatString = + String.format( + "create pipe p1" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.history.start-time'=%s)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + "%s", receiverIp, receiverPort); + + final List invalidStartTimes = + Arrays.asList("''", "null", "'null'", "'-1000-01-01T00:00:00'", "'2000-01-01T00:00:0'"); + for (final String invalidStartTime : invalidStartTimes) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(String.format(formatString, invalidStartTime)); + fail(); + } catch (final SQLException ignored) { + } + } + assertPipeCount(0); + + // Scenario 2: can not set 'extractor.history.enable' and 'extractor.realtime.enable' both to + // false + final String p2 = + String.format( + "create pipe p2" + + " with extractor (" + + "'extractor.history.enable'='false'," + + "'extractor.realtime.enable'='false')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p2); + fail(); + } catch (final SQLException ignored) { + } + assertPipeCount(0); + + // Scenario 3: test when 'extractor.history.start-time' is greater than + // 'extractor.history.end-time' + final String p3 = + String.format( + "create pipe p3" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.history.start-time'='2001.01.01T08:00:00'," + + "'extractor.history.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p3); + fail(); + } catch (final SQLException ignored) { + } + assertPipeCount(0); + + // Scenario 4: test when 'source.start-time' is greater than 'source.end-time' + final String p4 = + String.format( + "create pipe p4" + + " with source (" + + "'source.start-time'='2001.01.01T08:00:00'," + + "'source.end-time'='2000.01.01T08:00:00')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(p4); + fail(); + } catch (final SQLException ignored) { + } + assertPipeCount(0); + } + + @Test + public void testExtractorPatternMatch() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.nonAligned.1TS (time, s_float) values (now(), 0.5)", + "insert into root.nonAligned.100TS (time, s_float) values (now(), 0.5)", + "insert into root.nonAligned.1000TS (time, s_float) values (now(), 0.5)", + "insert into root.nonAligned.`1(TS)` (time, s_float) values (now(), 0.5)", + "insert into root.nonAligned.6TS.`6` (" + + "time, `s_float(1)`, `s_int(1)`, `s_double(1)`, `s_long(1)`, `s_text(1)`, `s_bool(1)`) " + + "values (now(), 0.5, 1, 1.5, 2, \"text1\", true)", + "insert into root.aligned.1TS (time, s_float) aligned values (now(), 0.5)", + "insert into root.aligned.100TS (time, s_float) aligned values (now(), 0.5)", + "insert into root.aligned.1000TS (time, s_float) aligned values (now(), 0.5)", + "insert into root.aligned.`1(TS)` (time, s_float) aligned values (now(), 0.5)", + "insert into root.aligned.6TS.`6` (" + + "time, `s_float(1)`, `s_int(1)`, `s_double(1)`, `s_long(1)`, `s_text(1)`, `s_bool(1)`) " + + "aligned values (now(), 0.5, 1, 1.5, 2, \"text1\", true)"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", null); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final List patterns = + Arrays.asList( + "root.db.nonExistPath", // match nothing + "root.nonAligned.1TS.s_float", + "root.nonAligned.6TS.`6`.`s_text(1)`", + "root.aligned.1TS", + "root.aligned.6TS.`6`.`s_int(1)`", + "root.aligned.`1(TS)`", + "root.nonAligned.`1(TS)`", + "root.aligned.6TS.`6`", // match device root.aligned.6TS.`6` + "root.nonAligned.6TS.`6`", // match device root.nonAligned.6TS.`6` + "root.nonAligned.`1`", // match nothing + "root.nonAligned.1", // match device root.nonAligned.1TS, 100TS and 100TS + "root.aligned.1" // match device root.aligned.1TS, 100TS and 100TS + ); + + final List expectedTimeseriesCount = + Arrays.asList(0, 1, 2, 3, 4, 5, 6, 11, 16, 16, 18, 20); + + for (int i = 0; i < patterns.size(); ++i) { + extractorAttributes.replace("extractor.pattern", patterns.get(i)); + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p" + i, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p" + i).getCode()); + // We add flush here because the pipe may be created on the new IoT leader + // and the old leader's data may come as an unclosed historical tsfile + // and is skipped flush when the pipe starts. In this case, the "waitForTsFileClose()" + // may not return until a flush is executed, namely the data transfer relies + // on a flush operation. + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + assertTimeseriesCountOnReceiver(receiverEnv, expectedTimeseriesCount.get(i)); + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show devices", + "Device,IsAligned,Template,TTL(ms),", + new HashSet<>( + Arrays.asList( + "root.nonAligned.1TS,false,null,INF,", + "root.nonAligned.100TS,false,null,INF,", + "root.nonAligned.1000TS,false,null,INF,", + "root.nonAligned.`1(TS)`,false,null,INF,", + "root.nonAligned.6TS.`6`,false,null,INF,", + "root.aligned.1TS,true,null,INF,", + "root.aligned.100TS,true,null,INF,", + "root.aligned.1000TS,true,null,INF,", + "root.aligned.`1(TS)`,true,null,INF,", + "root.aligned.6TS.`6`,true,null,INF,"))); + } + } + + @Test + public void testMatchingMultipleDatabases() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db1"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + assertTimeseriesCountOnReceiver(receiverEnv, 0); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1) values (1, 10)", + "insert into root.db2.d1 (time, at1) values (1, 20)", + "flush"))) { + return; + } + + extractorAttributes.replace("extractor.pattern", "root.db2"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + assertTimeseriesCountOnReceiver(receiverEnv, 2); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1) values (2, 11)", + "insert into root.db2.d1 (time, at1) values (2, 21)", + "flush"))) { + return; + } + + extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db1.d1.at1),count(root.db2.d1.at1),", + Collections.singleton("2,2,")); + } + } + + @Test + public void testHistoryAndRealtime() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (1, 10)", + "insert into root.db.d2 (time, at1) values (1, 20)", + "insert into root.db.d3 (time, at1) values (1, 30)", + "insert into root.db.d4 (time, at1) values (1, 40)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.pattern", "root.db.d2"); + extractorAttributes.put("extractor.history.enable", "false"); + extractorAttributes.put("extractor.realtime.enable", "true"); + TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + extractorAttributes.replace("extractor.pattern", "root.db.d3"); + extractorAttributes.replace("extractor.history.enable", "true"); + extractorAttributes.replace("extractor.realtime.enable", "false"); + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + + extractorAttributes.replace("extractor.pattern", "root.db.d4"); + extractorAttributes.replace("extractor.history.enable", "true"); + extractorAttributes.replace("extractor.realtime.enable", "true"); + status = + client.createPipe( + new TCreatePipeReq("p4", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p4").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (2, 11)", + "insert into root.db.d2 (time, at1) values (2, 21)", + "insert into root.db.d3 (time, at1) values (2, 31)", + "insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** where time <= 1", + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + Collections.singleton("1,0,1,")); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** where time >= 2", + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + Collections.singleton("2,1,0,")); + } + } + + @Test + public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", + "insert into root.db.d2 (time, at1)" + + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.history.enable", "true"); + // 1970-01-01T08:00:02+08:00 + extractorAttributes.put("extractor.history.start-time", "2000"); + extractorAttributes.put("extractor.history.end-time", "1970-01-01T08:00:04+08:00"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),", + Collections.singleton("3,")); + + extractorAttributes.remove("extractor.pattern"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d2.at1),", + Collections.singleton("3,3,")); + } + } + + @Test + public void testExtractorTimeRangeMatch() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // insert history data + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", + "insert into root.db.d2 (time, at1)" + + " values (6000, 6), (7000, 7), (8000, 8), (9000, 9), (10000, 10)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("source.inclusion", "data"); + extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); + extractorAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),", + Collections.singleton("3,")); + + // Insert realtime data that overlapped with time range + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d3 (time, at1)" + + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d3.at1),", + Collections.singleton("3,3,")); + + // Insert realtime data that does not overlap with time range + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d4 (time, at1)" + + " values (6000, 6), (7000, 7), (8000, 8), (9000, 9), (10000, 10)", + "flush"))) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d3.at1),", + Collections.singleton("3,3,")); + } + } + + @Test + public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", + "insert into root.db.d2 (time, at1)" + + " values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.pattern", "root.db.d1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); + // 1970-01-01T08:00:04+08:00 + extractorAttributes.put("source.end-time", "4000"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),", + Collections.singleton("3,")); + + extractorAttributes.remove("source.pattern"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.at1),count(root.db.d2.at1),", + Collections.singleton("3,3,")); + } + } + + @Test + public void testHistoryLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 1, extracted without parse + "insert into root.db.d1 (time, at1, at2)" + " values (1000, 1, 2), (2000, 3, 4)", + // TsFile 2, not extracted because pattern not overlapped + "insert into root.db1.d1 (time, at1, at2)" + " values (1000, 1, 2), (2000, 3, 4)", + "flush"))) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 3, not extracted because time range not overlapped + "insert into root.db.d1 (time, at1, at2)" + " values (3000, 1, 2), (4000, 3, 4)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.history.start-time", "1500"); + extractorAttributes.put("source.history.end-time", "2500"); + extractorAttributes.put("source.history.loose-range", "time, path"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** group by level=0", + "count(root.*.*.*),", + Collections.singleton("4,")); + } + } + + @Test + public void testRealtimeLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.realtime.loose-range", "time, path"); + extractorAttributes.put("source.start-time", "2000"); + extractorAttributes.put("source.end-time", "10000"); + extractorAttributes.put("source.realtime.mode", "batch"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1, at2)" + " values (1000, 1, 2), (3000, 3, 4)", + "flush"))) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1, at2)" + " values (1000, 1, 2), (3000, 3, 4)", + "flush"))) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + " values (5000, 1), (16000, 3)", + "insert into root.db.d1 (time, at1, at2)" + " values (5001, 1, 2), (6001, 3, 4)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(at1) from root.db.d1 where time >= 2000 and time <= 10000", + new HashMap() { + { + put("count(root.db.d1.at1)", "4"); + } + }); + } + } + + private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton(count + ",")); + } + + private void assertPipeCount(int count) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(count, showPipeResult.size()); + // for (TShowPipeInfo showPipeInfo : showPipeResult) { + // System.out.println(showPipeInfo); + // } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java new file mode 100644 index 0000000000000..9ee9df2d57187 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java @@ -0,0 +1,861 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueriesWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.executeQueryWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeLifeCycleIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testLifeCycleWithHistoryEnabled() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + expectedResSet.add("2,2.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + expectedResSet.add("3,3.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + } + + @Test + public void testLifeCycleWithHistoryDisabled() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.inclusion.exclusion", ""); + + extractorAttributes.put("extractor.history.enable", "false"); + // start-time and end-time should not work + extractorAttributes.put("extractor.history.start-time", "0001.01.01T00:00:00"); + extractorAttributes.put("extractor.history.end-time", "2100.01.01T00:00:00"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.ln", + "create timeseries root.db.d1.s2 with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.db.d1(time, s1) values (2, 2)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select s1 from root.db.d1", + "Time,root.db.d1.s1,", + Collections.singleton("2,2.0,")); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("2,")); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("2,")); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.ln0", + "create timeseries root.db.d1.s3 with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.db.d1(time, s1) values (3, 3)", + "flush"))) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "select s1 from root.db.d1", + "Time,root.db.d1.s1,", + Collections.singleton("2,2.0,")); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("2,")); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("2,")); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select s1 from root.db.d1", + "Time,root.db.d1.s1,", + new HashSet<>(Arrays.asList("2,2.0,", "3,3.0,"))); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("3,")); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("3,")); + } + } + + @Test + public void testLifeCycleLogMode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.mode", "log"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + expectedResSet.add("2,2.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + } + + @Test + public void testLifeCycleFileMode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.mode", "file"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + expectedResSet.add("2,2.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + } + + @Test + public void testLifeCycleHybridMode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.mode", "hybrid"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + expectedResSet.add("2,2.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + } + + @Test + public void testLifeCycleWithClusterRestart() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Set expectedResSet = new HashSet<>(); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + expectedResSet.add("1,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + expectedResSet.add("2,2.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient ignored = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + expectedResSet.add("3,3.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + } + + @Test + public void testReceiverRestartWhenTransferring() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final AtomicInteger succeedNum = new AtomicInteger(0); + final Thread t = + new Thread( + () -> { + try { + for (int i = 0; i < 100; ++i) { + if (TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + succeedNum.incrementAndGet(); + } + Thread.sleep(100); + } + } catch (InterruptedException ignored) { + } + }); + t.start(); + + try { + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + try { + t.interrupt(); + t.join(); + } catch (Throwable ignored) { + } + return; + } + + t.join(); + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton(succeedNum.get() + ",")); + } + } + + @Test + public void testReceiverAlreadyHaveTimeSeries() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + Thread.sleep(5000); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + } + } + + @Test + public void testDoubleLiving() throws Exception { + // Double living is two clusters with pipes connecting each other. + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + for (int i = 0; i < 100; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // Add this property to avoid to make self cycle. + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + for (int i = 100; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + for (int i = 200; i < 300; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // Add this property to avoid to make self cycle. + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + for (int i = 300; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + for (int i = 400; i < 500; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + for (int i = 500; i < 600; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + for (int i = 400; i < 600; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + + @Test + public void testPermission() { + createUser(senderEnv, "test", "test123"); + + assertNonQueryTestFail( + senderEnv, + "create pipe testPipe\n" + + "with connector (\n" + + " 'connector'='iotdb-thrift-connector',\n" + + " 'connector.ip'='127.0.0.1',\n" + + " 'connector.port'='6668'\n" + + ")", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + assertNonQueryTestFail( + senderEnv, + "drop pipe testPipe", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + assertTestFail( + senderEnv, + "show pipes", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + assertNonQueryTestFail( + senderEnv, + "start pipe testPipe", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + assertNonQueryTestFail( + senderEnv, + "stop pipe testPipe", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + + assertNonQueryTestFail( + senderEnv, + "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + assertNonQueryTestFail( + senderEnv, + "drop pipePlugin TestProcessor", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + assertTestFail( + senderEnv, + "show pipe plugins", + "803: No permissions for this operation, please add privilege USE_PIPE", + "test", + "test123"); + + grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.USE_PIPE); + + executeNonQueryWithRetry( + senderEnv, + "create pipe testPipe\n" + + "with connector (\n" + + " 'connector'='write-back-connector'\n" + + ")", + "test", + "test123"); + executeQueryWithRetry(senderEnv, "show pipes", "test", "test123"); + executeNonQueriesWithRetry( + senderEnv, + Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), + "test", + "test123"); + + assertNonQueryTestFail( + senderEnv, + "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", + "701: Untrusted uri xxx", + "test", + "test123"); + executeQueryWithRetry(senderEnv, "show pipe plugins", "test", "test123"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java index 4556416f5fbe3..53b5f27b0c418 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,11 +27,13 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -43,8 +45,9 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeProcessorIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeProcessorIT extends AbstractPipeDualTreeModelAutoIT { + @Before public void setUp() { MultiEnvFactory.createEnv(2); @@ -67,13 +70,14 @@ public void setUp() { .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); senderEnv.initClusterEnvironment(); receiverEnv.initClusterEnvironment(); } + @Ignore @Test public void testTumblingTimeSamplingProcessor() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java new file mode 100644 index 0000000000000..b286cc61365f6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java @@ -0,0 +1,494 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + } + + private void innerSetUp( + final String configNodeConsensus, + final String schemaRegionConsensus, + final String dataRegionConsensus, + final int configNodesNum, + final int dataNodesNum, + int schemaRegionReplicationFactor, + int dataRegionReplicationFactor) { + schemaRegionReplicationFactor = Math.min(schemaRegionReplicationFactor, dataNodesNum); + dataRegionReplicationFactor = Math.min(dataRegionReplicationFactor, dataNodesNum); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(dataRegionConsensus) + .setSchemaReplicationFactor(schemaRegionReplicationFactor) + .setDataReplicationFactor(dataRegionReplicationFactor); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(configNodeConsensus) + .setSchemaRegionConsensusProtocolClass(schemaRegionConsensus) + .setDataRegionConsensusProtocolClass(dataRegionConsensus) + .setSchemaReplicationFactor(schemaRegionReplicationFactor) + .setDataReplicationFactor(dataRegionReplicationFactor); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + } + + @Test + public void test1C1DWithRatisRatisIot() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test1C1DWithSimpleSimpleIot() throws Exception { + innerSetUp( + ConsensusFactory.SIMPLE_CONSENSUS, + ConsensusFactory.SIMPLE_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test1C1DWithRatisRatisSimple() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.SIMPLE_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 3, + 3, + 3, + 3); + doTest(); + } + + @Test + public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 3, + 3, + 3, + 2); + doTest(); + } + + @Test + public void testPipeOnBothSenderAndReceiver() throws Exception { + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(3, 3); + receiverEnv.initClusterEnvironment(1, 1); + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + } + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, "insert into root.db.d1(time, s1) values (2, 2)")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + } + } + + private void doTest() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + Thread.sleep(5000); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + } + } + + @Test + public void testSyncConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); + } + + @Test + public void testAsyncConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); + } + + @Test + public void testAirGapConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); + } + + private void doTestUseNodeUrls(String connectorName) throws Exception { + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1) + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(1, 1); + receiverEnv.initClusterEnvironment(1, 3); + + final StringBuilder nodeUrlsBuilder = new StringBuilder(); + for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { + if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { + // Use default port for convenience + nodeUrlsBuilder + .append(wrapper.getIp()) + .append(":") + .append(wrapper.getPipeAirGapReceiverPort()) + .append(","); + } else { + nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); + } + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Test mods transfer + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (1, 1)", + "insert into root.db.d1(time, s1) values (3, 1)", + "flush", + "delete from root.db.d1.s1 where time > 2"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", connectorName); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); + + extractorAttributes.put("source.inclusion", "all"); + extractorAttributes.put("source.mods.enable", "true"); + + // Test forced-log mode, in open releases this might be "file" + extractorAttributes.put("source.realtime.mode", "forced-log"); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(s1) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + + // Test metadata + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create timeseries root.db.d1.s2 with datatype=BOOLEAN,encoding=PLAIN", + "create database root.test1"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("2,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("2,")); + + // Test file mode + extractorAttributes.put("source.inclusion", "data"); + extractorAttributes.replace("source.realtime.mode", "file"); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(s1) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton("3,")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java new file mode 100644 index 0000000000000..d015e4641b730 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testPipeSwitchStatus() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p2") && o.state.equals("RUNNING"))); + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p3") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p3").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p2") && o.state.equals("RUNNING"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + } + } + + @Test + public void testPipeIllegallySwitchStatus() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), status.getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + } + } + + @Test + public void testDropPipeAndCreateAgain() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "drop database root.**")) { + return; + } + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals( + 0, + showPipeResult.stream() + .filter(info -> !info.id.startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) + .count()); + + status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.stream().filter((o) -> o.id.equals("p1")).count()); + } + } + + @Test + public void testWrongPipeName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipe("*").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("STOPPED"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java new file mode 100644 index 0000000000000..8d5d7097810a3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java @@ -0,0 +1,741 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeSyntaxIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testValidPipeName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final List validPipeNames = + Arrays.asList("Pipe_1", "null", "`33`", "`root`", "中文", "with"); + final List expectedPipeNames = + Arrays.asList("Pipe_1", "null", "33", "root", "中文", "with"); + for (final String pipeName : validPipeNames) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + pipeName, receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + for (final String pipeName : expectedPipeNames) { + Assert.assertTrue( + showPipeResult.stream() + .anyMatch((o) -> o.id.equals(pipeName) && o.state.equals("RUNNING"))); + } + + for (final String pipeName : validPipeNames) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("drop pipe %s", pipeName)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testRevertParameterOrder() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with extractor (" + + "'extractor.realtime.mode'='hybrid'," + + "'extractor.history.enable'='false') " + + " with connector (" + + "'connector.batch.enable'='false', " + + "'connector.port'='%s'," + + "'connector.ip'='%s'," + + "'connector'='iotdb-thrift-connector')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignore) { + } + } + + @Test + public void testRevertStageOrder() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with connector (" + + "'connector.batch.enable'='false', " + + "'connector.port'='%s'," + + "'connector.ip'='%s'," + + "'connector'='iotdb-thrift-connector') " + + " with extractor (" + + "'extractor.realtime.mode'='hybrid'," + + "'extractor.history.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testMissingStage() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create pipe p1"); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create pipe p2 with extractor ('extractor'='iotdb-extractor')"); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + "create pipe p3" + + " with extractor ('extractor'='iotdb-extractor')" + + " with processor ('processor'='do-nothing-processor')"); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p5" + + " with extractor ('extractor'='iotdb-extractor')" + + " with processor ('processor'='do-nothing-processor')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(2, showPipeResult.size()); + } + } + + @Test + public void testInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with extractor ()" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p2" + + " with extractor ('extractor'='invalid-param')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p3" + + " with extractor ()" + + " with processor ('processor'='invalid-param')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ()" + + " with processor ()" + + " with connector (" + + "'connector'='invalid-param'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + } + } + + @Test + public void testBrackets() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor1" + + " with extractor ('extractor'='iotdb-extractor')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor2" + + " with extractor (\"extractor\"=\"iotdb-extractor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor3" + + " with extractor ('extractor'=\"iotdb-extractor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor4" + + " with extractor (extractor=iotdb-extractor)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe extractor5" + + " with extractor ('extractor'=`iotdb-extractor`)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor1" + + " with processor ('processor'='do-nothing-processor')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor2" + + " with processor (\"processor\"=\"do-nothing-processor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor3" + + " with processor ('processor'=\"do-nothing-processor\")" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor4" + + " with processor (processor=do-nothing-processor)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe processor5" + + " with processor ('processor'=`do-nothing-processor`)" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe connector1" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe connector2" + + " with connector (" + + "\"connector\"=\"iotdb-thrift-connector\"," + + "\"connector.ip\"=\"%s\"," + + "\"connector.port\"=\"%s\"," + + "\"connector.batch.enable\"=\"false\")", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe connector3" + + " with connector (" + + "'connector'=\"iotdb-thrift-connector\"," + + "\"connector.ip\"='%s'," + + "'connector.port'=\"%s\"," + + "\"connector.batch.enable\"='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe connector4" + + " with connector (" + + "connector=iotdb-thrift-connector," + + "connector.ip=%s," + + "connector.port=%s," + + "connector.batch.enable=false)", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe connector5" + + " with connector (" + + "'connector'=`iotdb-thrift-connector`," + + "'connector.ip'=`%s`," + + "'connector.port'=`%s`," + + "'connector.batch.enable'=`false`)", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(9, showPipeResult.size()); + } + } + + @Test + public void testShowPipeWithWrongPipeName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + connectorAttributes.replace("connector.batch.enable", "true"); + + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(3, showPipeResult.size()); + + showPipeResult = client.showPipe(new TShowPipeReq().setPipeName("p1")).pipeInfoList; + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); + + // Show all pipes whose connector is also used by p1. + // p1 and p2 share the same connector parameters, so they have the same connector. + showPipeResult = + client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); + Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); + } + } + + @Test + public void testInclusionPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Empty inclusion + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p2" + + " with extractor ('extractor.inclusion'='schema, auth.role', 'extractor.inclusion.exclusion'='all')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + // Invalid inclusion + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p3" + + " with extractor ('extractor.inclusion'='wrong')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + // Invalid exclusion + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.inclusion.exclusion'='wrong')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException ignored) { + } + + // Valid + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.inclusion'='all', 'extractor.inclusion.exclusion'='schema.database.drop, auth.role')" + + " with processor ()" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + } + } + + @Test + public void testValidPipeWithoutWithSink() { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create pipe p1('sink'='do-nothing-sink')"); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java new file mode 100644 index 0000000000000..608aae2b4aee0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBTreePatternFormatIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testPrefixPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", + "insert into root.db.d2(time, s) values (1, 1)", + "insert into root.db2.d1(time, s) values (1, 1)"))) { + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s,root.db.d1.s1,", expectedResSet); + } + } + + @Test + public void testIotdbPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", + "insert into root.db.d2(time, s) values (1, 1)", + "insert into root.db2.d1(time, s) values (1, 1)"))) { + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.**.d1.s*"); + // When path is set, pattern should be ignored + extractorAttributes.put("extractor.pattern", "root"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + expectedResSet); + } + } + + @Test + public void testIotdbPatternWithLegacySyntax() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", + "insert into root.db.d2(time, s) values (1, 1)", + "insert into root.db2.d1(time, s) values (1, 1)"))) { + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.**.d1.s*"); + extractorAttributes.put("extractor.pattern.format", "iotdb"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,1.0,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + expectedResSet); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java new file mode 100644 index 0000000000000..34b42c35c141c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeAutoConflictIT extends AbstractPipeDualTreeModelAutoIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testDoubleLivingAutoConflict() throws Exception { + // Double living is two clusters each with a pipe connecting to the other. + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + for (int i = 0; i < 100; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "all"); + extractorAttributes.put("source.inclusion.exclusion", ""); + // Add this property to avoid making self cycle. + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + for (int i = 100; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + for (int i = 200; i < 300; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "all"); + extractorAttributes.put("source.inclusion.exclusion", ""); + // Add this property to avoid to make self cycle. + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + for (int i = 300; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + for (int i = 400; i < 500; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + for (int i = 500; i < 600; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + for (int i = 400; i < 600; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + } + + @Test + public void testDoubleLivingAutoConflictTemplate() throws Exception { + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "all"); + extractorAttributes.put("source.inclusion.exclusion", ""); + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.inclusion", "all"); + extractorAttributes.put("source.inclusion.exclusion", ""); + extractorAttributes.put("source.forwarding-pipe-requests", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // Auto extend s1 + "create schema template t1 (s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", + "create database root.db", + "set device template t1 to root.db"))) { + return; + } + + for (int i = 0; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + + for (int i = 200; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet); + } + + @Test + public void testAutoManualCreateRace() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, "create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN")) { + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, + "create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN tags (tag3=v3) attributes (attr4=v4)")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeSeries", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + Collections.singleton( + "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,RLE,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,")); + } + } + + @Test + public void testHistoricalActivationRace() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.sg_aligned", + "create device template aligned_template aligned (s0 int32, s1 int64, s2 float, s3 double, s4 boolean, s5 text)", + "set device template aligned_template to root.sg_aligned.device_aligned", + "create timeseries using device template on root.sg_aligned.device_aligned.d10", + "create timeseries using device template on root.sg_aligned.device_aligned.d12", + "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5) values (1706659200,1706659200,10,20.245,25.24555,true,''),(1706662800,null,1706662800,20.241,25.24111,false,'2'),(1706666400,3,null,20.242,25.24222,true,'3'),(1706670000,4,40,null,35.5474,true,'4'),(1706670600,5,1706670600000,20.246,null,false,'5'),(1706671200,6,60,20.248,25.24888,null,'6'),(1706671800,7,1706671800,20.249,25.24999,false,null),(1706672400,8,80,1245.392,75.51234,false,'8'),(1706672600,9,90,2345.397,2285.58734,false,'9'),(1706673000,10,100,20.241,25.24555,false,'10'),(1706673600,11,110,3345.394,4105.544,false,'11'),(1706674200,12,1706674200,30.245,35.24555,false,'12'),(1706674800,13,130,5.39,125.51234,false,'13'),(1706675400,14,1706675400,5.39,135.51234,false,'14'),(1706676000,15,150,5.39,145.51234,false,'15'),(1706676600,16,160,5.39,155.51234,false,'16'),(1706677200,17,170,5.39,165.51234,false,'17'),(1706677600,18,180,5.39,175.51234,false,'18'),(1706677800,19,190,5.39,185.51234,false,'19'),(1706678000,20,200,5.39,195.51234,false,'20'),(1706678200,21,210,5.39,null,false,'21')", + "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5) values (-1,1,10,5.39,5.51234,false,'negative')", + "insert into root.sg_aligned.device_aligned.d11(time, s0, s1, s2,s3,s4,s5) values (-1,-11,-110,-5.39,-5.51234,false,'activate:1')", + "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5,s6) values(1706678800,1,1706678800,5.39,5.51234,false,'add:s6',32);"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count devices", "count(devices),", Collections.singleton("3,")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java new file mode 100644 index 0000000000000..493a991ae1b12 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; +import static org.awaitility.Awaitility.await; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeAutoDropIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testAutoDropInHistoricalTransfer() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Collections.singletonList("insert into root.db.d1(time, s1) values (1, 1)"))) { + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.mode", "query"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + try (final Connection connection = makeItCloseQuietly(senderEnv.getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement()); ) { + ResultSet result = statement.executeQuery("show pipes"); + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(600, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + int pipeNum = 0; + while (result.next()) { + if (!result + .getString(ColumnHeaderConstant.ID) + .contains(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + pipeNum++; + } + } + Assert.assertEquals(0, pipeNum); + } catch (Exception e) { + Assert.fail(); + } + }); + } + } + } + + @Test + public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Collections.singletonList( + "insert into root.db.d1(time, s1) values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)"))) { + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.mode", "query"); + extractorAttributes.put("extractor.start-time", "1970-01-01T08:00:02+08:00"); + extractorAttributes.put("extractor.end-time", "1970-01-01T08:00:04+08:00"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("3,")); + + try (final Connection connection = makeItCloseQuietly(senderEnv.getConnection()); + final Statement statement = makeItCloseQuietly(connection.createStatement()); ) { + ResultSet result = statement.executeQuery("show pipes"); + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(600, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + int pipeNum = 0; + while (result.next()) { + if (!result + .getString(ColumnHeaderConstant.ID) + .contains(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + pipeNum++; + } + } + Assert.assertEquals(0, pipeNum); + } catch (Exception e) { + Assert.fail(); + } + }); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java new file mode 100644 index 0000000000000..ff827d6332b64 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java @@ -0,0 +1,1047 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionRoleType; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeClusterIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setDataReplicationFactor(2) + .setSchemaReplicationFactor(3) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(3, 3, 180); + receiverEnv.initClusterEnvironment(3, 3, 180); + } + + @Test + public void testMachineDowntimeAsync() { + testMachineDowntime("iotdb-thrift-connector"); + } + + @Test + public void testMachineDowntimeSync() { + testMachineDowntime("iotdb-thrift-sync-connector"); + } + + private void testMachineDowntime(String sink) { + StringBuilder a = new StringBuilder(); + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + a.append(nodeWrapper.getIp()).append(":").append(nodeWrapper.getPort()); + a.append(","); + } + a.deleteCharAt(a.length() - 1); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 1, senderEnv); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", + "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("capture.tree", "true"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", sink); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.node-urls", a.toString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + receiverEnv.getDataNodeWrapper(0).stop(); + + // Ensure that the kill -9 operation is completed + Thread.sleep(5000); + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + if (!nodeWrapper.isAlive()) { + continue; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + nodeWrapper, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,"), + 600); + } + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.db.d1(time, s1) values (now(), 3)", "flush"))) { + return; + } + + } catch (Exception e) { + fail(e.getMessage()); + } + + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + if (!nodeWrapper.isAlive()) { + continue; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + nodeWrapper, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("3,"), + 600); + return; + } + } + + @Test + public void testWithAllParametersInLogMode() throws Exception { + testWithAllParameters("log"); + } + + @Test + public void testWithAllParametersInFileMode() throws Exception { + testWithAllParameters("file"); + } + + @Test + public void testWithAllParametersInHybridMode() throws Exception { + testWithAllParameters("hybrid"); + } + + public void testWithAllParameters(final String realtimeMode) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", + "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", + "flush"))) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("extractor.pattern", "root.db.d1"); + extractorAttributes.put("extractor.history.enable", "true"); + extractorAttributes.put("extractor.history.start-time", "2010-01-01T08:00:00+08:00"); + extractorAttributes.put("extractor.history.end-time", "2010-01-02T08:00:00+08:00"); + extractorAttributes.put("extractor.realtime.enable", "true"); + extractorAttributes.put("extractor.realtime.mode", realtimeMode); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "root"); + connectorAttributes.put("connector.password", "root"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.db.d1(time, s1) values (now(), 3)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + } + } + + @Test + public void testPipeAfterDataRegionLeaderStop() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + final AtomicInteger leaderPort = new AtomicInteger(-1); + final TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq()); + showRegionResp + .getRegionInfoList() + .forEach( + regionInfo -> { + if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { + leaderPort.set(regionInfo.getClientRpcPort()); + } + }); + + int leaderIndex = -1; + for (int i = 0; i < 3; ++i) { + if (senderEnv.getDataNodeWrapper(i).getPort() == leaderPort.get()) { + leaderIndex = i; + try { + senderEnv.shutdownDataNode(i); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + try { + TimeUnit.SECONDS.sleep(1); + } catch (final InterruptedException ignored) { + } + try { + senderEnv.startDataNode(i); + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + } + } + if (leaderIndex == -1) { // ensure the leader is stopped + fail(); + } + + if (!TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( + senderEnv, + senderEnv.getDataNodeWrapper(leaderIndex), + Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Create a new pipe and write new data + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d2"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d2(time, s1) values (1, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d2", + "count(root.db.d2.s1),", + Collections.singleton("1,")); + } + } + + @Test + public void testPipeAfterRegisterNewDataNode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"))) { + return; + } + + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + final DataNodeWrapper newDataNode = + senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); + if (!TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( + senderEnv, + newDataNode, + Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"))) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // create a new pipe and write new data + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d2"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Arrays.asList("insert into root.db.d2(time, s1) values (1, 1)", "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d2", + "count(root.db.d2.s1),", + Collections.singleton("1,")); + } + } + + @Test + public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final Thread t = + new Thread( + () -> { + for (int i = 0; i < 30; ++i) { + try { + client.createPipe( + new TCreatePipeReq("p" + i, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + } catch (final TException e) { + // Not sure if the "createPipe" has succeeded + e.printStackTrace(); + return; + } + try { + Thread.sleep(100); + } catch (final Exception ignored) { + } + } + }); + t.start(); + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + t.join(); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(30, showPipeResult.size()); + } + } + + @Test + public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final AtomicInteger succeedNum = new AtomicInteger(0); + final Thread t = + new Thread( + () -> { + try { + for (int i = 0; i < 100; ++i) { + if (TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + succeedNum.incrementAndGet(); + } + Thread.sleep(100); + } + } catch (final InterruptedException ignored) { + } + }); + t.start(); + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + t.join(); + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton(succeedNum.get() + ",")); + + try { + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + } catch (final Throwable e) { + e.printStackTrace(); + } + } + } + + @Test + public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + int succeedNum = 0; + for (int i = 0; i < 100; ++i) { + if (TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + succeedNum++; + } + } + + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton(succeedNum + ",")); + + try { + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + } catch (final Throwable e) { + e.printStackTrace(); + } + } + } + + @Ignore( + "Currently ignore this test because this test intends to test the behaviour when the sender has" + + " a temporary node joined and then removed, but in reality it just tears it down. In this" + + " circumstance the IT may fail. However, the \"remove\" method is currently not provided thus" + + " we ignore this test now.") + @Test + public void testNewDataNodeFailureParallelToTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + int succeedNum = 0; + for (int i = 0; i < 100; ++i) { + if (TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i * 1000))) { + succeedNum++; + } + } + + try { + senderEnv.registerNewDataNode(false); + senderEnv.startDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.d1", + "count(root.db.d1.s1),", + Collections.singleton(succeedNum + ",")); + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(1, showPipeResult.size()); + } + } + + @Test + public void testSenderRestartWhenTransferring() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + int succeedNum = 0; + for (int i = 0; i < 100; ++i) { + if (TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i * 1000))) { + succeedNum++; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try { + TestUtils.restartCluster(senderEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton(succeedNum + ",")); + } + + @Test + public void testConcurrentlyCreatePipeOfSameName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final AtomicInteger successCount = new AtomicInteger(0); + final List threads = new ArrayList<>(); + for (int i = 0; i < 10; ++i) { + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successCount.incrementAndGet(); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + + for (Thread t : threads) { + t.join(); + } + Assert.assertEquals(1, successCount.get()); + + successCount.set(0); + for (int i = 0; i < 10; ++i) { + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = client.dropPipe("p1"); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successCount.incrementAndGet(); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + for (final Thread t : threads) { + t.join(); + } + + // Assert at least 1 drop operation succeeds + Assert.assertTrue(successCount.get() >= 1); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testCreate10PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(10); + } + + @Test + public void testCreate50PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(50); + } + + @Test + public void testCreate100PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(100); + } + + private void testCreatePipesWithSameConnector(final int pipeCount) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final AtomicInteger successCount = new AtomicInteger(0); + final List threads = new ArrayList<>(); + for (int i = 0; i < pipeCount; ++i) { + final int finalI = i; + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p" + finalI, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + successCount.incrementAndGet(); + } catch (final InterruptedException e) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + for (final Thread t : threads) { + t.join(); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals(successCount.get(), showPipeResult.size()); + showPipeResult = + client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + Assert.assertEquals(successCount.get(), showPipeResult.size()); + } + } + + @Test + public void testNegativeTimestamp() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (0, 1)", + "insert into root.db.d1(time, s1) values (-1, 2)", + "insert into root.db.d1(time, s1) values (1960-01-02T10:00:00+08:00, 2)", + "flush"))) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("3,")); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // Test the correctness of insertRowsNode transmission + "insert into root.db.d1(time, s1) values (-122, 3)", + "insert into root.db.d1(time, s1) values (-123, 3), (now(), 3)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("6,")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java index e6dfd37a307dd..3e75838dd3d5f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java @@ -17,16 +17,18 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -39,8 +41,14 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } @Test public void testBasicCreatePipeIfNotExists() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java new file mode 100644 index 0000000000000..37a542e6aa9fb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + // Override to enable air-gap + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testCompression1() throws Exception { + doTest("iotdb-thrift-connector", "stream", true, "snappy"); + } + + @Test + public void testCompression2() throws Exception { + doTest("iotdb-thrift-connector", "batch", true, "snappy, lzma2"); + } + + @Test + public void testCompression3() throws Exception { + doTest("iotdb-thrift-sync-connector", "stream", false, "snappy, snappy"); + } + + @Test + public void testCompression4() throws Exception { + doTest("iotdb-thrift-sync-connector", "batch", true, "gzip, zstd"); + } + + @Test + public void testCompression5() throws Exception { + doTest("iotdb-air-gap-connector", "stream", false, "lzma2, lz4"); + } + + @Test + public void testCompression6() throws Exception { + doTest("iotdb-air-gap-connector", "batch", true, "lzma2"); + } + + private void doTest( + String connectorType, String realtimeMode, boolean useBatchMode, String compressionTypes) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = + connectorType.contains("air-gap") + ? receiverDataNode.getPipeAirGapReceiverPort() + : receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", + "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", + "flush"))) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("extractor.realtime.mode", realtimeMode); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", connectorType); + connectorAttributes.put("connector.batch.enable", useBatchMode ? "true" : "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "root"); + connectorAttributes.put("connector.password", "root"); + connectorAttributes.put("connector.compressor", compressionTypes); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("2,")); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (now(), 3)", + "insert into root.db.d1(time, s1) values (now(), 4)", + "insert into root.db.d1(time, s1) values (now(), 5)", + "insert into root.db.d1(time, s1) values (now(), 6)", + "insert into root.db.d1(time, s1) values (now(), 7)", + "insert into root.db.d1(time, s1) values (now(), 8)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("8,")); + } + } + + @Test + public void testZstdCompressorLevel() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s1) values (1, 1)", + "insert into root.db.d1(time, s2) values (1, 1)", + "insert into root.db.d1(time, s3) values (1, 1)", + "insert into root.db.d1(time, s4) values (1, 1)", + "insert into root.db.d1(time, s5) values (1, 1)", + "flush"))) { + return; + } + + // Create 5 pipes with different zstd compression levels, p4 and p5 should fail. + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p1" + + " with extractor ('extractor.pattern'='root.db.d1.s1')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='3')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p2" + + " with extractor ('extractor.pattern'='root.db.d1.s2')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='22')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p3" + + " with extractor ('extractor.pattern'='root.db.d1.s3')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='-131072')", + receiverIp, receiverPort)); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.pattern'='root.db.d1.s4')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='-131073')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException e) { + // Make sure the error message in IoTDBConnector.java is returned + Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); + } + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe p5" + + " with extractor ('extractor.pattern'='root.db.d1.s5')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='23')", + receiverIp, receiverPort)); + fail(); + } catch (SQLException e) { + // Make sure the error message in IoTDBConnector.java is returned + Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); + } + + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertEquals( + 3, + showPipeResult.stream() + .filter(info -> !info.id.startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) + .count()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("3,")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java new file mode 100644 index 0000000000000..f9cfc7f9fe30a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeIdempotentIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + // All the schema operations must be under the same database to + // be in the same region, therefore a non-idempotent operation can block the next one + // and fail the IT + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + // Limit the schemaRegion number to 1 to guarantee the after sql executed on the same region + // of the tested idempotent sql. + .setDefaultSchemaRegionGroupNumPerDatabase(1) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testCreateTimeSeriesIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN", + "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("2,")); + } + + @Test + public void testCreateAlignedTimeSeriesIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "CREATE ALIGNED TIMESERIES root.ln.wf01.GPS(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY)", + "create timeSeries root.ln.wf01.wt01.status1(status) with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("3,")); + } + + @Test + public void testCreateTimeSeriesWithAliasIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "create timeSeries root.ln.wf01.wt01.status0(status0) with datatype=BOOLEAN,encoding=PLAIN", + "create timeSeries root.ln.wf01.wt01.status1(status1) with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("2,")); + } + + @Test + public void testInternalCreateTimeSeriesIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "insert into root.ln.wf01.wt01(time, status0) values(now(), false); flush;", + "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("2,")); + } + + @Test + public void testAlterTimeSeriesAddTagIdempotent() throws Exception { + testIdempotent( + Collections.singletonList( + "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN"), + "ALTER timeSeries root.ln.wf01.wt01.status0 ADD TAGS tag3=v3;", + "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("2,")); + } + + @Test + public void testAlterTimeSeriesAddAttrIdempotent() throws Exception { + testIdempotent( + Collections.singletonList( + "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN"), + "ALTER timeSeries root.ln.wf01.wt01.status0 ADD ATTRIBUTES attr1=newV1;", + "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("2,")); + } + + @Test + public void testAlterTimeSeriesRenameIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create timeSeries root.ln.wf01.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeSeries root.ln.wf01.wt01.status0 ADD ATTRIBUTES attr1=newV1;"), + "ALTER timeSeries root.ln.wf01.wt01.status0 RENAME attr1 TO newAttr1;", + "create timeSeries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN", + "count timeSeries", + "count(timeseries),", + Collections.singleton("2,")); + } + + @Test + public void testDeleteTimeSeriesIdempotent() throws Exception { + testIdempotent( + Collections.singletonList( + "create timeSeries root.ln.wf01.wt01.status0(status0) with datatype=BOOLEAN,encoding=PLAIN"), + "delete timeSeries root.ln.wf01.wt01.status0", + "create database root.sg", + "count databases", + "count,", + Collections.singleton("2,")); + } + + @Test + public void testCreateTemplateIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testExtendTemplateIdempotent() throws Exception { + testIdempotent( + Collections.singletonList( + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)"), + "alter schema template t1 add (rest FLOAT encoding=RLE, FLOAT2 TEXT encoding=PLAIN compression=SNAPPY)", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testDropTemplateIdempotent() throws Exception { + testIdempotent( + Collections.singletonList( + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)"), + "drop schema template t1", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testSetTemplateIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", + "create database root.sg1"), + "set schema template t1 to root.sg1", + "create database root.sg2", + "count databases", + "count,", + Collections.singleton("2,")); + } + + @Test + public void testUnsetTemplateIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", + "create database root.sg1", + "set schema template t1 to root.sg1"), + "unset schema template t1 from root.sg1", + "create database root.sg2", + "count databases", + "count,", + Collections.singleton("2,")); + } + + @Test + public void testActivateTemplateIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", + "create database root.sg1", + "set schema template t1 to root.sg1"), + "create timeSeries using device template on root.sg1.d1", + "create timeSeries using device template on root.sg1.d2", + "count timeSeries", + "count(timeseries),", + Collections.singleton("6,")); + } + + @Test + public void testDeactivateTemplateIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", + "create database root.sg1", + "set schema template t1 to root.sg1", + "create timeSeries using device template on root.sg1.d1", + "create timeSeries using device template on root.sg1.d2"), + "delete timeSeries of schema template t1 from root.sg1.*", + "create database root.sg2", + "count databases", + "count,", + Collections.singleton("2,")); + } + + @Test + public void testCreateDatabaseIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "create database root.sg1", + "create database root.sg2", + "count databases", + "count,", + Collections.singleton("2,")); + } + + @Test + public void testAlterDatabaseIdempotent() throws Exception { + testIdempotent( + Collections.singletonList("create database root.sg1"), + "ALTER DATABASE root.sg1 WITH SCHEMA_REGION_GROUP_NUM=2, DATA_REGION_GROUP_NUM=3;", + "create database root.sg2", + "count databases", + "count,", + Collections.singleton("2,")); + } + + @Test + public void testDropDatabaseIdempotent() throws Exception { + testIdempotent( + Collections.singletonList("create database root.sg1"), + "drop database root.sg1", + "create database root.sg2", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testCreateUserIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "create user `ln_write_user` 'write_pwd'", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testCreateRoleIdempotent() throws Exception { + testIdempotent( + Collections.emptyList(), + "create role `test`", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testGrantRoleToUserIdempotent() throws Exception { + testIdempotent( + Arrays.asList("create user `ln_write_user` 'write_pwd'", "create role `test`"), + "grant role test to ln_write_user", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testRevokeUserIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create user `ln_write_user` 'write_pwd'", + "GRANT READ_DATA, WRITE_DATA ON root.t1.** TO USER ln_write_user;"), + "REVOKE READ_DATA, WRITE_DATA ON root.t1.** FROM USER ln_write_user;", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testRevokeRoleIdempotent() throws Exception { + testIdempotent( + Arrays.asList("create role `test`", "GRANT READ ON root.** TO ROLE test;"), + "REVOKE READ ON root.** FROM ROLE test;", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testRevokeRoleFromUserIdempotent() throws Exception { + testIdempotent( + Arrays.asList( + "create user `ln_write_user` 'write_pwd'", + "create role `test`", + "grant role test to ln_write_user"), + "revoke role test from ln_write_user", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testDropUserIdempotent() throws Exception { + testIdempotent( + Collections.singletonList("create user `ln_write_user` 'write_pwd'"), + "drop user `ln_write_user`", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + @Test + public void testDropRoleIdempotent() throws Exception { + testIdempotent( + Collections.singletonList("create role `test`"), + "drop role test", + "create database root.sg1", + "count databases", + "count,", + Collections.singleton("1,")); + } + + // Table model + + private void testIdempotent( + final List beforeSqlList, + final String testSql, + final String afterSql, + final String afterSqlQuery, + final String expectedHeader, + final Set expectedResSet) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.inclusion.exclusion", ""); + extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, beforeSqlList)) { + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, testSql)) { + return; + } + + // Create an idempotent conflict, after sql shall be executed on the same region as testSql + if (!TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Arrays.asList(testSql, afterSql))) { + return; + } + + // Assume that the afterSql is executed on receiverEnv + TestUtils.assertDataEventuallyOnEnv(receiverEnv, afterSqlQuery, expectedHeader, expectedResSet); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java new file mode 100644 index 0000000000000..765104e1f5233 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeNullValueIT extends AbstractPipeDualTreeModelAutoIT { + + // Test dimensions: + // 1. is or not aligned + // 2. is or not parsed + // 3. session insertRecord, session insertTablet, SQL insert + // 4. partial null, all null + // 5. one row or more (TODO) + // 6. more data types (TODO) + + private enum InsertType { + SESSION_INSERT_RECORD, + SESSION_INSERT_TABLET, + SQL_INSERT, + } + + private static final Map> INSERT_NULL_VALUE_MAP = new HashMap<>(); + + private static final List CREATE_TIMESERIES_SQL = + Arrays.asList( + "create timeseries root.sg.d1.s0 with datatype=float", + "create timeseries root.sg.d1.s1 with datatype=float"); + + private static final List CREATE_ALIGNED_TIMESERIES_SQL = + Collections.singletonList("create aligned timeseries root.sg.d1(s0 float, s1 float)"); + + private final String deviceId = "root.sg.d1"; + private final List measurements = Arrays.asList("s0", "s1"); + private final List types = Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT); + + private final List partialNullValues = Arrays.asList(null, 25.34F); + private final List allNullValues = Arrays.asList(null, null); + + private Tablet partialNullTablet; + private Tablet allNullTablet; + + private void constructTablet() { + final MeasurementSchema[] schemas = new MeasurementSchema[2]; + for (int i = 0; i < schemas.length; i++) { + schemas[i] = new MeasurementSchema(measurements.get(i), types.get(i)); + } + + final BitMap[] bitMapsForPartialNull = new BitMap[2]; + bitMapsForPartialNull[0] = new BitMap(1); + bitMapsForPartialNull[0].markAll(); + bitMapsForPartialNull[1] = new BitMap(1); + + final BitMap[] bitMapsForAllNull = new BitMap[2]; + bitMapsForAllNull[0] = new BitMap(1); + bitMapsForAllNull[0].markAll(); + bitMapsForAllNull[1] = new BitMap(1); + bitMapsForAllNull[1].markAll(); + + final Object[] valuesForPartialNull = new Object[2]; + valuesForPartialNull[0] = new float[] {0F}; + valuesForPartialNull[1] = new float[] {25.34F}; + + final Object[] valuesForAllNull = new Object[2]; + valuesForAllNull[0] = new float[] {0F}; + valuesForAllNull[1] = new float[] {0F}; + + partialNullTablet = + new Tablet( + deviceId, + Arrays.asList(schemas), + new long[] {3}, + valuesForPartialNull, + bitMapsForPartialNull, + 1); + + allNullTablet = + new Tablet( + deviceId, + Arrays.asList(schemas), + new long[] {4}, + valuesForAllNull, + bitMapsForAllNull, + 1); + } + + @Override + @Before + public void setUp() { + super.setUp(); + + constructTablet(); + + // Initialize INSERT_NULL_VALUE_MAP + INSERT_NULL_VALUE_MAP.put( + InsertType.SESSION_INSERT_RECORD, + (isAligned) -> { + try { + try (final ISession session = senderEnv.getSessionConnection()) { + if (isAligned) { + session.insertAlignedRecord(deviceId, 3, measurements, types, partialNullValues); + session.insertAlignedRecord(deviceId, 4, measurements, types, allNullValues); + } else { + session.insertRecord(deviceId, 3, measurements, types, partialNullValues); + session.insertRecord(deviceId, 4, measurements, types, allNullValues); + } + } catch (StatementExecutionException e) { + fail(e.getMessage()); + } + } catch (IoTDBConnectionException e) { + fail(e.getMessage()); + } + }); + + INSERT_NULL_VALUE_MAP.put( + InsertType.SESSION_INSERT_TABLET, + (isAligned) -> { + try { + try (final ISession session = senderEnv.getSessionConnection()) { + if (isAligned) { + session.insertAlignedTablet(partialNullTablet); + session.insertAlignedTablet(allNullTablet); + } else { + session.insertTablet(partialNullTablet); + session.insertTablet(allNullTablet); + } + } catch (StatementExecutionException e) { + fail(e.getMessage()); + } + } catch (IoTDBConnectionException e) { + fail(e.getMessage()); + } + }); + + INSERT_NULL_VALUE_MAP.put( + InsertType.SQL_INSERT, + (isAligned) -> { + // Partial null + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + isAligned + ? Collections.singletonList( + "insert into root.sg.d1(time, s0, s1) aligned values (3, null, 25.34)") + : Collections.singletonList( + "insert into root.sg.d1(time, s0, s1) values (3, null, 25.34)"))) { + fail(); + } + // All null + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + isAligned + ? Collections.singletonList( + "insert into root.sg.d1(time, s0, s1) aligned values (4, null, null)") + : Collections.singletonList( + "insert into root.sg.d1(time, s0, s1) values (4, null, null)"))) { + fail(); + } + }); + } + + private void testInsertNullValueTemplate( + final InsertType insertType, final boolean isAligned, final boolean withParsing) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + if (withParsing) { + extractorAttributes.put("start-time", "1970-01-01T08:00:00.000+08:00"); + extractorAttributes.put("end-time", "1970-01-01T09:00:00.000+08:00"); + extractorAttributes.put("extractor.pattern", "root.sg.d1"); + } + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("test", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + receiverEnv, isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL)) { + fail(); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL)) { + fail(); + } + + INSERT_NULL_VALUE_MAP.get(insertType).accept(isAligned); + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + fail(); + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.sg.d1.s0),count(root.sg.d1.s1),", + Collections.singleton("0,1,")); + } + + // ---------------------- // + // Scenario 1: SQL Insert // + // ---------------------- // + @Test + public void testSQLInsertWithParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, true); + } + + @Test + public void testSQLInsertWithoutParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, false); + } + + @Test + public void testSQLInsertAlignedWithParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, true); + } + + @Test + public void testSQLInsertAlignedWithoutParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, false); + } + + // --------------------------------- // + // Scenario 2: Session Insert Record // + // --------------------------------- // + @Test + public void testSessionInsertRecordWithParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, false, true); + } + + @Test + public void testSessionInsertRecordWithoutParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, false, false); + } + + @Test + public void testSessionInsertRecordAlignedWithParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, true, true); + } + + @Test + public void testSessionInsertRecordAlignedWithoutParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, true, false); + } + + // --------------------------------- // + // Scenario 3: Session Insert Tablet // + // --------------------------------- // + @Test + public void testSessionInsertTabletWithParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, true); + } + + @Test + public void testSessionInsertTabletWithoutParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, false); + } + + @Test + public void testSessionInsertTabletAlignedWithParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, true); + } + + @Test + public void testSessionInsertTabletAlignedWithoutParsing() throws Exception { + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, false); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java new file mode 100644 index 0000000000000..0b929e4daf88b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeWithLoadIT extends AbstractPipeDualTreeModelAutoIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Disable sender compaction to test mods + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + /** + * Test that when the receiver loads data from TsFile, it will not load timeseries that are + * completed deleted by mods. + * + * @throws Exception + */ + @Test + public void testReceiverNotLoadDeletedTimeseries() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + // Enable mods transfer + extractorAttributes.put("source.mods.enable", "true"); + + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Generate TsFile and mods on sender. There are 6 time-series in total. + // Time-series not affected by mods: d1.s1, d2.s1 + // Time-series partially deleted by mods: d1.s2, d3.s1 + // Time-series completely deleted by mods: d1.s3, d4.s1 (should not be loaded by receiver) + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, s1, s2, s3) values (1, 1, 1, 1), (3, 3, 3, 3)", + "insert into root.db.d2 (time, s1) values (1, 1), (3, 3)", + "insert into root.db.d3 (time, s1) values (1, 1), (3, 3)", + "insert into root.db.d4 (time, s1) values (1, 1), (3, 3)", + "flush", + "delete from root.db.d1.s2 where time <= 2", + "delete from root.db.d1.s3 where time >= 1 and time <= 3", + "delete from root.db.d3.** where time <= 2", + "delete from root.db.d4.** where time >= 1 and time <= 3", + "flush"))) { + return; + } + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("4,")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java new file mode 100644 index 0000000000000..09845735dc134 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class PipeNowFunctionIT extends AbstractPipeDualTreeModelAutoIT { + + @Override + @Before + public void setUp() { + super.setUp(); + } + + @Test + public void testPipeNowFunction() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Map extractorAttributes = new HashMap<>(); + Map processorAttributes = new HashMap<>(); + Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.start-time", "now"); + extractorAttributes.put("source.end-time", "now"); + extractorAttributes.put("source.history.start-time", "now"); + extractorAttributes.put("source.history.end-time", "now"); + extractorAttributes.put("source.history.enable", "true"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + extractorAttributes.clear(); + extractorAttributes.put("start-time", "now"); + extractorAttributes.put("end-time", "now"); + extractorAttributes.put("history.start-time", "now"); + extractorAttributes.put("history.end-time", "now"); + extractorAttributes.put("history.enable", "true"); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + extractorAttributes.clear(); + extractorAttributes.put("extractor.start-time", "now"); + extractorAttributes.put("extractor.end-time", "now"); + extractorAttributes.put("extractor.history.start-time", "now"); + extractorAttributes.put("extractor.history.end-time", "now"); + extractorAttributes.put("history.enable", "true"); + + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + extractorAttributes.clear(); + extractorAttributes.put("extractor.start-time", "now"); + extractorAttributes.put("extractor.end-time", "now"); + extractorAttributes.put("extractor.history.start-time", "now"); + extractorAttributes.put("extractor.history.end-time", "now"); + client.alterPipe( + new TAlterPipeReq() + .setPipeName("p1") + .setExtractorAttributes(extractorAttributes) + .setIsReplaceAllExtractorAttributes(false) + .setProcessorAttributes(new HashMap<>()) + .setIsReplaceAllProcessorAttributes(false) + .setConnectorAttributes(new HashMap<>()) + .setIsReplaceAllConnectorAttributes(false)); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue( + showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); + + extractorAttributes.clear(); + extractorAttributes.put("start-time", "now"); + extractorAttributes.put("end-time", "now"); + extractorAttributes.put("history.start-time", "now"); + extractorAttributes.put("history.end-time", "now"); + client.alterPipe( + new TAlterPipeReq() + .setPipeName("p1") + .setExtractorAttributes(extractorAttributes) + .setIsReplaceAllExtractorAttributes(false) + .setProcessorAttributes(new HashMap<>()) + .setIsReplaceAllProcessorAttributes(false) + .setConnectorAttributes(new HashMap<>()) + .setIsReplaceAllConnectorAttributes(false)); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); + } + } + + @Test + public void testTreeModeSQLSupportNowFunc() { + doTest(BaseEnv.TREE_SQL_DIALECT); + } + + private void doTest(String dialect) { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final String p1 = + String.format( + "create pipe p1" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(dialect); + final Statement statement = connection.createStatement()) { + statement.execute(p1); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + final String p2 = + String.format( + "create pipe p2" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(dialect); + final Statement statement = connection.createStatement()) { + statement.execute(p2); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + final String p3 = + String.format( + "create pipe p3" + + " with extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(dialect); + final Statement statement = connection.createStatement()) { + statement.execute(p3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + String alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(dialect); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(dialect); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + alterP3 = + "alter pipe p3" + + " modify source (" + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(dialect); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java new file mode 100644 index 0000000000000..581d363cdb5c5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractPipeDualTreeModelManualIT { + + protected BaseEnv senderEnv; + protected BaseEnv receiverEnv; + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + setupConfig(); + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + protected void setupConfig() { + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + } + + @After + public final void tearDown() { + senderEnv.cleanClusterEnvironment(); + receiverEnv.cleanClusterEnvironment(); + } + + protected static void awaitUntilFlush(BaseEnv env) { + Awaitility.await() + .atMost(1, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + if (!TestUtils.tryExecuteNonQueryWithRetry(env, "flush")) { + return false; + } + return env.getDataNodeWrapperList().stream() + .anyMatch( + wrapper -> { + int fileNum = 0; + File sequence = new File(buildDataPath(wrapper, true)); + File unsequence = new File(buildDataPath(wrapper, false)); + if (sequence.exists() && sequence.listFiles() != null) { + fileNum += Objects.requireNonNull(sequence.listFiles()).length; + } + if (unsequence.exists() && unsequence.listFiles() != null) { + fileNum += Objects.requireNonNull(unsequence.listFiles()).length; + } + return fileNum > 0; + }); + }); + } + + private static String buildDataPath(DataNodeWrapper wrapper, boolean isSequence) { + String nodePath = wrapper.getNodePath(); + return nodePath + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + "datanode" + + File.separator + + IoTDBConstant.DATA_FOLDER_NAME + + File.separator + + (isSequence ? IoTDBConstant.SEQUENCE_FOLDER_NAME : IoTDBConstant.UNSEQUENCE_FOLDER_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java similarity index 92% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 4515536c2533c..4bc4505c1b6f0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeInclusionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT { @Test public void testPureSchemaInclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -180,6 +180,7 @@ public void testAuthInclusionWithPattern() throws Exception { senderEnv, Arrays.asList( "create user `ln_write_user` 'write_pwd'", + "grant manage_database,manage_user,manage_role,use_trigger,use_udf,use_cq,use_pipe on root.** to USER ln_write_user with grant option", "GRANT READ_DATA, WRITE_DATA ON root.** TO USER ln_write_user;"))) { return; } @@ -189,7 +190,16 @@ public void testAuthInclusionWithPattern() throws Exception { "LIST PRIVILEGES OF USER ln_write_user", "ROLE,PATH,PRIVILEGES,GRANT OPTION,", new HashSet<>( - Arrays.asList(",root.ln.**,READ_DATA,false,", ",root.ln.**,WRITE_DATA,false,"))); + Arrays.asList( + ",root.**,MANAGE_USER,true,", + ",root.**,MANAGE_ROLE,true,", + ",root.**,USE_TRIGGER,true,", + ",root.**,USE_UDF,true,", + ",root.**,USE_CQ,true,", + ",root.**,USE_PIPE,true,", + ",root.**,MANAGE_DATABASE,true,", + ",root.ln.**,READ_DATA,false,", + ",root.ln.**,WRITE_DATA,false,"))); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java index dbb1810922865..63bef4ff759c2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeManualConflictIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeManualConflictIT extends AbstractPipeDualTreeModelManualIT { @Test public void testDoubleLivingTimeseries() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java new file mode 100644 index 0000000000000..c351aad274c4c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualTreeModelManualIT { + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setDefaultSchemaRegionGroupNumPerDatabase(1) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(3, 3); + } + + @Test + public void testTemplateInclusion() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.ln", + "create database root.db", + "set ttl to root.ln 3600000", + "create user `thulab` 'passwd'", + "create role `admin`", + "grant role `admin` to `thulab`", + "grant read on root.** to role `admin`", + "create schema template t1 (temperature FLOAT encoding=RLE, status BOOLEAN encoding=PLAIN compression=SNAPPY)", + "set schema template t1 to root.ln.wf01", + "set schema template t1 to root.db.wf01", + "create timeseries using schema template on root.ln.wf01.wt01", + "create timeseries using schema template on root.db.wf01.wt01", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + // Insert large timestamp to avoid deletion by ttl + "insert into root.ln.wf01.wt01(time, temperature, status) values (1800000000000, 23, true)"))) { + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "data, schema"); + extractorAttributes.put("extractor.inclusion.exclusion", "schema.timeseries.ordinary"); + extractorAttributes.put("extractor.path", "root.ln.**"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "list user", + ColumnHeaderConstant.USER + ",", + Collections.singleton("root,")); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "list role", ColumnHeaderConstant.ROLE + ",", Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show databases", + "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionOrigin,TimePartitionInterval,", + // Receiver's SchemaReplicationFactor/DataReplicationFactor shall be 3/2 regardless of the + // sender + Collections.singleton("root.ln,3,2,0,604800000,")); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.ln.wf01.wt01.temperature,root.ln.wf01.wt01.status,", + Collections.singleton("1800000000000,23.0,true,")); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "create timeseries using schema template on root.ln.wf01.wt02")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("4,")); + } + } + + @Test + public void testAuthInclusion() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.ln", + "set ttl to root.ln 3600000", + "create user `thulab` 'passwd'", + "create role `admin`", + "grant role `admin` to `thulab`", + "grant read on root.** to role `admin`", + "grant manage_database,manage_user,manage_role,use_trigger,use_udf,use_cq,use_pipe on root.** to role `admin`;", + "create schema template t1 (temperature FLOAT encoding=RLE, status BOOLEAN encoding=PLAIN compression=SNAPPY)", + "set schema template t1 to root.ln.wf01", + "create timeseries using schema template on root.ln.wf01.wt01", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + // Insert large timestamp to avoid deletion by ttl + "insert into root.ln.wf01.wt01(time, temperature, status) values (1800000000000, 23, true)"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "auth"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "list user of role `admin`", + ColumnHeaderConstant.USER + ",", + Collections.singleton("thulab,")); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "list privileges of role `admin`", + ColumnHeaderConstant.ROLE + + "," + + ColumnHeaderConstant.SCOPE + + "," + + ColumnHeaderConstant.PRIVILEGES + + "," + + ColumnHeaderConstant.GRANT_OPTION + + ",", + new HashSet<>( + Arrays.asList( + "admin,,MANAGE_USER,false,", + "admin,,MANAGE_ROLE,false,", + "admin,,USE_TRIGGER,false,", + "admin,,USE_UDF,false,", + "admin,,USE_CQ,false,", + "admin,,USE_PIPE,false,", + "admin,,MANAGE_DATABASE,false,", + "admin,root.**,READ_DATA,false,", + "admin,root.**,READ_SCHEMA,false,"))); + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show databases", + "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionOrigin,TimePartitionInterval,", + Collections.emptySet()); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "select * from root.**", "Time", Collections.emptySet()); + + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "CREATE ROLE test")) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "list role", + ColumnHeaderConstant.ROLE + ",", + new HashSet<>(Arrays.asList("admin,", "test,"))); + } + } + + @Test + public void testTimeSeriesInclusion() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.sg", + "create timeseries root.sg.a.b int32", + "create aligned timeseries root.sg.`apache|timecho-tag-attr`.d1(s1 INT32 tags(tag1=v1, tag2=v2) attributes(attr1=v1, attr2=v2), s2 DOUBLE tags(tag3=v3, tag4=v4) attributes(attr3=v3, attr4=v4))"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "schema"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeseries", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + new HashSet<>( + Arrays.asList( + "root.sg.a.b,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.`apache|timecho-tag-attr`.d1.s1,null,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},{\"attr2\":\"v2\",\"attr1\":\"v1\"},null,null,BASE,", + "root.sg.`apache|timecho-tag-attr`.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,{\"tag4\":\"v4\",\"tag3\":\"v3\"},{\"attr4\":\"v4\",\"attr3\":\"v3\"},null,null,BASE,"))); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java index de386f7f3dc26..00281253abb02 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +41,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualTreeModelManualIT { @Override @Before public void setUp() { @@ -59,8 +59,8 @@ public void setUp() { .setSchemaReplicationFactor(3); // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); senderEnv.initClusterEnvironment(3, 3, 180); receiverEnv.initClusterEnvironment(); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java index 76cd4a90e8b87..ec9930d2642b6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -38,8 +38,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMetaRestartIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMetaRestartIT extends AbstractPipeDualTreeModelManualIT { @Test public void testAutoRestartSchemaTask() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java index 769b940c78d25..8c210f64638d6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -39,8 +39,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMultiSchemaRegionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMultiSchemaRegionIT extends AbstractPipeDualTreeModelManualIT { @Test public void testMultiSchemaRegion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java new file mode 100644 index 0000000000000..52320d52bd957 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipePermissionIT extends AbstractPipeDualTreeModelManualIT { + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setDefaultSchemaRegionGroupNumPerDatabase(1) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(3, 3); + } + + @Test + public void testWithSyncConnector() throws Exception { + testWithConnector("iotdb-thrift-sync-connector"); + } + + @Test + public void testWithAsyncConnector() throws Exception { + testWithConnector("iotdb-thrift-async-connector"); + } + + private void testWithConnector(final String connector) throws Exception { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + receiverEnv, + Arrays.asList( + "create user `thulab` 'passwd'", + "create role `admin`", + "grant role `admin` to `thulab`", + "grant WRITE, READ, MANAGE_DATABASE, MANAGE_USER on root.** to role `admin`"))) { + return; + } + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create user user 'passwd'", + "create timeseries root.ln.wf02.wt01.temperature with datatype=INT64,encoding=PLAIN", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf02.wt01(time, temperature, status) values (1800000000000, 23, true)"))) { + fail(); + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + + connectorAttributes.put("connector", connector); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.username", "thulab"); + connectorAttributes.put("connector.password", "passwd"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "list user", + "User,", + new HashSet<>(Arrays.asList("root,", "user,", "thulab,"))); + final Set expectedResSet = new HashSet<>(); + expectedResSet.add( + "root.ln.wf02.wt01.temperature,null,root.ln,INT64,PLAIN,LZ4,null,null,null,null,BASE,"); + expectedResSet.add( + "root.ln.wf02.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,null,null,null,null,BASE,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeseries", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + expectedResSet); + expectedResSet.clear(); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.ln.wf02.wt01.temperature,root.ln.wf02.wt01.status,", + Collections.singleton("1800000000000,23,true,")); + } + } + + @Test + public void testNoPermission() throws Exception { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + receiverEnv, + Arrays.asList( + "create user `thulab` 'passwd'", + "create role `admin`", + "grant role `admin` to `thulab`", + "grant READ, MANAGE_DATABASE on root.ln.** to role `admin`"))) { + return; + } + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create user someUser 'passwd'", + "create database root.noPermission", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN"))) { + fail(); + return; + } + awaitUntilFlush(senderEnv); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + + connectorAttributes.put("connector", "iotdb-thrift-async-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.username", "thulab"); + connectorAttributes.put("connector.password", "passwd"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("1,")); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show timeseries", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + Collections.emptySet()); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "list user", "User,", Collections.singleton("root,")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java new file mode 100644 index 0000000000000..55b5f72314daf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java @@ -0,0 +1,482 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeReqAutoSliceIT extends AbstractPipeDualTreeModelManualIT { + private static final int generateDataSize = 10; + + @Override + protected void setupConfig() { + super.setupConfig(); + senderEnv.getConfig().getCommonConfig().setPipeConnectorRequestSliceThresholdBytes(4); + receiverEnv.getConfig().getCommonConfig().setPipeConnectorRequestSliceThresholdBytes(4); + } + + @Test + public void insertTablet() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertTablet(tablet); + }, + false); + } + + @Ignore + @Test + public void insertTabletReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertTablet(tablet); + }, + true); + } + + @Ignore + @Test + public void insertAlignedTablet() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertAlignedTablet(tablet); + }, + false); + } + + @Ignore + @Test + public void insertAlignedTabletReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertAlignedTablet(tablet); + }, + true); + } + + @Ignore + @Test + public void insertRecordsReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + true); + } + + @Ignore + @Test + public void insertRecord() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i).toArray()); + } + }, + false); + } + + @Ignore + @Test + public void insertRecordReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i).toArray()); + } + }, + true); + } + + @Ignore + @Test + public void insertAlignedRecord() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertAlignedRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i)); + } + }, + false); + } + + @Ignore + @Test + public void insertAlignedRecordReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertAlignedRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i)); + } + }, + true); + } + + @Ignore + @Test + public void insertRecords() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + false); + } + + @Ignore + @Test + public void insertAlignedRecords() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertAlignedRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + false); + } + + @Ignore + @Test + public void insertAlignedRecordsReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertAlignedRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + true); + } + + @Ignore + @Test + public void insertStringRecordsOfOneDevice() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + false); + } + + @Ignore + @Test + public void insertStringRecordsOfOneDeviceReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + true); + } + + @Ignore + @Test + public void insertAlignedStringRecordsOfOneDevice() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertAlignedStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + false); + } + + @Ignore + @Test + public void insertAlignedStringRecordsOfOneDeviceReceiveByTsFile() { + prepareReqAutoSliceTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertAlignedStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + true); + } + + private void prepareReqAutoSliceTest( + CheckedTriConsumer consumer, boolean isTsFile) { + Tablet tablet = createTablet(); + createTimeSeries(); + try (ISession senderSession = senderEnv.getSessionConnection(); + ISession receiverSession = receiverEnv.getSessionConnection()) { + if (isTsFile) { + consumer.accept(senderSession, receiverSession, tablet); + senderSession.executeNonQueryStatement("flush"); + Thread.sleep(2000); + createPipe(senderSession, true); + } else { + createPipe(senderSession, false); + Thread.sleep(2000); + consumer.accept(senderSession, receiverSession, tablet); + senderSession.executeNonQueryStatement("flush"); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + verify(tablet); + } + + private void createPipe(ISession session, boolean isTsFile) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement( + String.format( + "create pipe test" + + " with source ('source'='iotdb-source','source.path'='root.test.**')" + + " with sink ('sink'='iotdb-thrift-sync-sink','node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", + receiverEnv.getIP(), receiverEnv.getPort(), isTsFile ? "tsfile" : "tablet")); + } + + private int[] createTestDataForInt32() { + int[] data = new int[generateDataSize]; + Random random = new Random(); + for (int i = 0; i < generateDataSize; i++) { + data[i] = random.nextInt(); + } + return data; + } + + private long[] createTestDataForInt64() { + long[] data = new long[generateDataSize]; + long time = System.currentTimeMillis(); + for (int i = 0; i < generateDataSize; i++) { + data[i] = time + i; + } + return data; + } + + private void verify(Tablet tablet) { + HashSet set = new HashSet<>(); + for (int i = 0; i < generateDataSize; i++) { + set.add( + String.format( + "%d,%d,%d,", + tablet.getTimestamp(i), (int) tablet.getValue(i, 0), (int) tablet.getValue(i, 1))); + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.test.** ORDER BY time ASC", + "Time,root.test.db.temperature,root.test.db.status,", + set, + 20); + } + + private void createTimeSeries() { + List timeSeriesCreation = + Arrays.asList( + "create timeseries root.test.db.status with datatype=INT32,encoding=PLAIN", + "create timeseries root.test.db.temperature with datatype=INT32,encoding=PLAIN"); + TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, timeSeriesCreation); + TestUtils.tryExecuteNonQueriesWithRetry(receiverEnv, timeSeriesCreation); + } + + private Tablet createTablet() { + long[] timestamp = createTestDataForInt64(); + int[] temperature = createTestDataForInt32(); + int[] status = createTestDataForInt32(); + + Object[] objects = new Object[2]; + objects[0] = temperature; + objects[1] = status; + + List measurementSchemas = new ArrayList<>(2); + measurementSchemas.add(new MeasurementSchema("temperature", TSDataType.INT32)); + measurementSchemas.add(new MeasurementSchema("status", TSDataType.INT32)); + + BitMap[] bitMaps = new BitMap[2]; + for (int i = 0; i < bitMaps.length; i++) { + bitMaps[i] = new BitMap(generateDataSize); + } + + return new Tablet( + "root.test.db", measurementSchemas, timestamp, objects, bitMaps, generateDataSize); + } + + private List getTimestampList(Tablet tablet) { + long[] timestamps = tablet.getTimestamps(); + List data = new ArrayList<>(timestamps.length); + for (long timestamp : timestamps) { + data.add(timestamp); + } + return data; + } + + private Pair>, List>> getMeasurementSchemasAndType( + Tablet tablet) { + List> schemaData = new ArrayList<>(tablet.getRowSize()); + List> typeData = new ArrayList<>(tablet.getRowSize()); + List measurementSchemas = new ArrayList<>(tablet.getSchemas().size()); + List types = new ArrayList<>(tablet.getRowSize()); + for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { + measurementSchemas.add(measurementSchema.getMeasurementName()); + types.add(measurementSchema.getType()); + } + + for (int i = 0; i < tablet.getRowSize(); i++) { + schemaData.add(measurementSchemas); + typeData.add(types); + } + + return new Pair<>(schemaData, typeData); + } + + private List getDeviceID(Tablet tablet) { + List data = new ArrayList<>(tablet.getRowSize()); + for (int i = 0; i < tablet.getRowSize(); i++) { + data.add(tablet.getDeviceId()); + } + return data; + } + + private List> generateTabletInsertRecordForTable(final Tablet tablet) { + List> insertRecords = new ArrayList<>(tablet.getRowSize()); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + for (int i = 0; i < tablet.getRowSize(); i++) { + List insertRecord = new ArrayList<>(); + for (int j = 0; j < schemas.size(); j++) { + switch (schemas.get(j).getType()) { + case INT64: + case TIMESTAMP: + insertRecord.add(((long[]) values[j])[i]); + break; + case INT32: + insertRecord.add(((int[]) values[j])[i]); + break; + } + } + insertRecords.add(insertRecord); + } + + return insertRecords; + } + + private List> generateTabletInsertStrRecordForTable(Tablet tablet) { + List> insertRecords = new ArrayList<>(tablet.getRowSize()); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + for (int i = 0; i < tablet.getRowSize(); i++) { + List insertRecord = new ArrayList<>(); + for (int j = 0; j < schemas.size(); j++) { + switch (schemas.get(j).getType()) { + case INT64: + insertRecord.add(String.valueOf(((long[]) values[j])[i])); + break; + case INT32: + insertRecord.add(String.valueOf(((int[]) values[j])[i])); + break; + } + } + insertRecords.add(insertRecord); + } + + return insertRecords; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java new file mode 100644 index 0000000000000..fd44c3d33836c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -0,0 +1,810 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModelManualIT { + private static final int generateDataSize = 100; + + @Test + public void insertTablet() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertTablet(tablet); + }, + false); + } + + @Test + public void insertTabletReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertTablet(tablet); + }, + true); + } + + @Test + public void insertAlignedTablet() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertAlignedTablet(tablet); + }, + false); + } + + @Test + public void insertAlignedTabletReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + senderSession.insertAlignedTablet(tablet); + }, + true); + } + + @Test + public void insertRecordsReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + true); + } + + @Test + public void insertRecord() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i).toArray()); + } + }, + false); + } + + @Test + public void insertRecordReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i).toArray()); + } + }, + true); + } + + @Test + public void insertAlignedRecord() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertAlignedRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i)); + } + }, + false); + } + + @Test + public void insertAlignedRecordReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + for (int i = 0; i < values.size(); i++) { + senderSession.insertAlignedRecord( + tablet.getDeviceId(), + timestamps.get(i), + pair.left.get(i), + pair.right.get(i), + values.get(i)); + } + }, + true); + } + + @Test + public void insertRecords() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + false); + } + + @Test + public void insertAlignedRecords() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertAlignedRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + false); + } + + @Test + public void insertAlignedRecordsReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertRecordForTable(tablet); + senderSession.insertAlignedRecords( + getDeviceID(tablet), timestamps, pair.left, pair.right, values); + }, + true); + } + + @Test + public void insertStringRecordsOfOneDevice() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + false); + } + + @Test + public void insertStringRecordsOfOneDeviceReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + true); + } + + @Test + public void insertAlignedStringRecordsOfOneDevice() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertAlignedStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + false); + } + + @Test + public void insertAlignedStringRecordsOfOneDeviceReceiveByTsFile() { + prepareTypeConversionTest( + (ISession senderSession, ISession receiverSession, Tablet tablet) -> { + List timestamps = getTimestampList(tablet); + Pair>, List>> pair = + getMeasurementSchemasAndType(tablet); + List> values = generateTabletInsertStrRecordForTable(tablet); + senderSession.insertAlignedStringRecordsOfOneDevice( + tablet.getDeviceId(), timestamps, pair.left, values); + }, + true); + } + + private SessionDataSet query( + ISession session, List measurementSchemas, String deviceId) + throws IoTDBConnectionException, StatementExecutionException { + String sql = "select "; + StringBuffer param = new StringBuffer(); + for (IMeasurementSchema schema : measurementSchemas) { + param.append(schema.getMeasurementName()); + param.append(','); + } + sql = sql + param.substring(0, param.length() - 1); + sql = sql + " from " + deviceId + " ORDER BY time ASC"; + return session.executeQueryStatement(sql); + } + + private void prepareTypeConversionTest( + CheckedTriConsumer consumer, boolean isTsFile) { + List> measurementSchemas = + generateMeasurementSchemas(); + + // Generate createTimeSeries in sender and receiver + String uuid = RandomStringUtils.random(8, true, false); + for (Pair pair : measurementSchemas) { + createTimeSeries( + uuid.toString(), pair.left.getMeasurementName(), pair.left.getType().name(), senderEnv); + createTimeSeries( + uuid.toString(), + pair.right.getMeasurementName(), + pair.right.getType().name(), + receiverEnv); + } + + try (ISession senderSession = senderEnv.getSessionConnection(); + ISession receiverSession = receiverEnv.getSessionConnection()) { + Tablet tablet = generateTabletAndMeasurementSchema(measurementSchemas, "root.test." + uuid); + if (isTsFile) { + // Send TsFile data to receiver + consumer.accept(senderSession, receiverSession, tablet); + Thread.sleep(2000); + createDataPipe(uuid, true); + senderSession.executeNonQueryStatement("flush"); + } else { + // Send Tablet data to receiver + createDataPipe(uuid, false); + Thread.sleep(2000); + // The actual implementation logic of inserting data + consumer.accept(senderSession, receiverSession, tablet); + senderSession.executeNonQueryStatement("flush"); + } + + // Verify receiver data + long timeoutSeconds = 600; + List> expectedValues = + generateTabletResultSetForTable(tablet, measurementSchemas); + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + validateResultSet( + query(receiverSession, tablet.getSchemas(), tablet.getDeviceId()), + expectedValues, + tablet.getTimestamps()); + } catch (Exception e) { + fail(); + } + }); + senderSession.close(); + receiverSession.close(); + tablet.reset(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void createTimeSeries(String diff, String measurementID, String dataType, BaseEnv env) { + String timeSeriesCreation = + String.format( + "create timeseries root.test.%s.%s with datatype=%s,encoding=PLAIN", + diff, measurementID, dataType); + TestUtils.tryExecuteNonQueriesWithRetry(env, Collections.singletonList(timeSeriesCreation)); + } + + private void createDataPipe(String diff, boolean isTSFile) { + String sql = + String.format( + "create pipe test%s" + + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')" + + " with processor ('processor'='do-nothing-processor')" + + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", + diff, + isTSFile ? "file" : "forced-log", + !isTSFile, + isTSFile, + receiverEnv.getIP(), + receiverEnv.getPort(), + isTSFile ? "tsfile" : "tablet"); + TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList(sql)); + } + + private void validateResultSet( + SessionDataSet dataSet, List> values, long[] timestamps) + throws IoTDBConnectionException, StatementExecutionException { + int index = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + List fields = record.getFields(); + + assertEquals(record.getTimestamp(), timestamps[index]); + List rowValues = values.get(index++); + for (int i = 0; i < fields.size(); i++) { + Field field = fields.get(i); + switch (field.getDataType()) { + case INT64: + case TIMESTAMP: + assertEquals(field.getLongV(), (long) rowValues.get(i)); + break; + case DATE: + assertEquals(field.getDateV(), rowValues.get(i)); + break; + case BLOB: + assertEquals(field.getBinaryV(), rowValues.get(i)); + break; + case TEXT: + case STRING: + assertEquals(field.getStringValue(), rowValues.get(i)); + break; + case INT32: + assertEquals(field.getIntV(), (int) rowValues.get(i)); + break; + case DOUBLE: + assertEquals(0, Double.compare(field.getDoubleV(), (double) rowValues.get(i))); + break; + case FLOAT: + assertEquals(0, Float.compare(field.getFloatV(), (float) rowValues.get(i))); + break; + } + } + } + assertEquals(values.size(), index); + } + + private boolean[] createTestDataForBoolean() { + boolean[] data = new boolean[generateDataSize]; + Random random = new Random(); + for (int i = 0; i < data.length; i++) { + data[i] = random.nextBoolean(); + } + return data; + } + + private int[] createTestDataForInt32() { + int[] data = new int[generateDataSize]; + Random random = new Random(); + for (int i = 0; i < data.length; i++) { + data[i] = random.nextInt(); + } + return data; + } + + private long[] createTestDataForInt64() { + long[] data = new long[generateDataSize]; + Random random = new Random(); + for (int i = 0; i < data.length; i++) { + data[i] = random.nextLong(); + } + return data; + } + + private float[] createTestDataForFloat() { + float[] data = new float[generateDataSize]; + Random random = new Random(); + for (int i = 0; i < data.length; i++) { + data[i] = random.nextFloat(); + } + return data; + } + + private double[] createTestDataForDouble() { + double[] data = new double[generateDataSize]; + Random random = new Random(); + for (int i = 0; i < data.length; i++) { + data[i] = random.nextDouble(); + } + return data; + } + + private long[] createTestDataForTimestamp() { + long[] data = new long[generateDataSize]; + long time = new Date().getTime(); + for (int i = 0; i < data.length; i++) { + data[i] = time + i; + } + return data; + } + + private LocalDate[] createTestDataForDate() { + LocalDate[] data = new LocalDate[generateDataSize]; + int year = 2023; + int month = 1; + int day = 1; + for (int i = 0; i < data.length; i++) { + data[i] = DateUtils.parseIntToLocalDate(year * 10000 + (month * 100) + day); + // update + day++; + if (day > 28) { + day = 1; + month++; + if (month > 12) { + month = 1; + year++; + } + } + } + return data; + } + + private Binary[] createTestDataForString() { + String[] stringData = { + "Hello", + "Hello World!", + "This is a test.", + "IoTDB Hello World!!!!", + "IoTDB is an excellent time series database!!!!!!!!!", + "12345678910!!!!!!!!", + "123456", + "1234567.123213", + "21232131.21", + "enable = true", + "true", + "false", + "12345678910", + "123231232132131233213123123123123123131312", + "123231232132131233213123123123123123131312.212312321312312", + }; + Binary[] data = new Binary[generateDataSize]; + for (int i = 0; i < data.length; i++) { + data[i] = + new Binary(stringData[(i % stringData.length)].getBytes(TSFileConfig.STRING_CHARSET)); + } + return data; + } + + private List getTimestampList(Tablet tablet) { + long[] timestamps = tablet.getTimestamps(); + List data = new ArrayList<>(timestamps.length); + for (long timestamp : timestamps) { + data.add(timestamp); + } + return data; + } + + private Pair>, List>> getMeasurementSchemasAndType( + Tablet tablet) { + List> schemaData = new ArrayList<>(tablet.getRowSize()); + List> typeData = new ArrayList<>(tablet.getRowSize()); + List measurementSchemas = new ArrayList<>(tablet.getSchemas().size()); + List types = new ArrayList<>(tablet.getRowSize()); + for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { + measurementSchemas.add(measurementSchema.getMeasurementName()); + types.add(measurementSchema.getType()); + } + + for (int i = 0; i < tablet.getRowSize(); i++) { + schemaData.add(measurementSchemas); + typeData.add(types); + } + + return new Pair<>(schemaData, typeData); + } + + private List getDeviceID(Tablet tablet) { + List data = new ArrayList<>(tablet.getRowSize()); + for (int i = 0; i < tablet.getRowSize(); i++) { + data.add(tablet.getDeviceId()); + } + return data; + } + + private List> generateTabletResultSetForTable( + final Tablet tablet, List> pairs) { + List> insertRecords = new ArrayList<>(tablet.getRowSize()); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + for (int i = 0; i < tablet.getRowSize(); i++) { + List insertRecord = new ArrayList<>(); + for (int j = 0; j < schemas.size(); j++) { + TSDataType sourceType = pairs.get(j).left.getType(); + TSDataType targetType = pairs.get(j).right.getType(); + Object value = null; + switch (sourceType) { + case INT64: + case TIMESTAMP: + value = ValueConverter.convert(sourceType, targetType, ((long[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case INT32: + value = ValueConverter.convert(sourceType, targetType, ((int[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case DOUBLE: + value = ValueConverter.convert(sourceType, targetType, ((double[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case FLOAT: + value = ValueConverter.convert(sourceType, targetType, ((float[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case DATE: + value = + ValueConverter.convert( + sourceType, + targetType, + DateUtils.parseDateExpressionToInt(((LocalDate[]) values[j])[i])); + insertRecord.add(convert(value, targetType)); + break; + case TEXT: + case STRING: + value = ValueConverter.convert(sourceType, targetType, ((Binary[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case BLOB: + value = ValueConverter.convert(sourceType, targetType, ((Binary[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + case BOOLEAN: + value = ValueConverter.convert(sourceType, targetType, ((boolean[]) values[j])[i]); + insertRecord.add(convert(value, targetType)); + break; + } + } + insertRecords.add(insertRecord); + } + + return insertRecords; + } + + private Object convert(Object value, TSDataType targetType) { + switch (targetType) { + case DATE: + return DateUtils.parseIntToLocalDate((Integer) value); + case TEXT: + case STRING: + return new String(((Binary) value).getValues(), TSFileConfig.STRING_CHARSET); + } + return value; + } + + private List> generateTabletInsertRecordForTable(final Tablet tablet) { + List> insertRecords = new ArrayList<>(tablet.getRowSize()); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + for (int i = 0; i < tablet.getRowSize(); i++) { + List insertRecord = new ArrayList<>(); + for (int j = 0; j < schemas.size(); j++) { + switch (schemas.get(j).getType()) { + case INT64: + case TIMESTAMP: + insertRecord.add(((long[]) values[j])[i]); + break; + case INT32: + insertRecord.add(((int[]) values[j])[i]); + break; + case DOUBLE: + insertRecord.add(((double[]) values[j])[i]); + break; + case FLOAT: + insertRecord.add(((float[]) values[j])[i]); + break; + case DATE: + insertRecord.add(((LocalDate[]) values[j])[i]); + break; + case TEXT: + case STRING: + insertRecord.add( + new String(((Binary[]) values[j])[i].getValues(), TSFileConfig.STRING_CHARSET)); + break; + case BLOB: + insertRecord.add(((Binary[]) values[j])[i]); + break; + case BOOLEAN: + insertRecord.add(((boolean[]) values[j])[i]); + break; + } + } + insertRecords.add(insertRecord); + } + + return insertRecords; + } + + private List> generateTabletInsertStrRecordForTable(Tablet tablet) { + List> insertRecords = new ArrayList<>(tablet.getRowSize()); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + for (int i = 0; i < tablet.getRowSize(); i++) { + List insertRecord = new ArrayList<>(); + for (int j = 0; j < schemas.size(); j++) { + switch (schemas.get(j).getType()) { + case INT64: + insertRecord.add(String.valueOf(((long[]) values[j])[i])); + break; + case TIMESTAMP: + insertRecord.add( + RpcUtils.formatDatetime("default", "ms", ((long[]) values[j])[i], ZoneOffset.UTC)); + break; + case INT32: + insertRecord.add(String.valueOf(((int[]) values[j])[i])); + break; + case DOUBLE: + insertRecord.add(String.valueOf(((double[]) values[j])[i])); + break; + case FLOAT: + insertRecord.add(String.valueOf(((float[]) values[j])[i])); + break; + case DATE: + insertRecord.add(((LocalDate[]) values[j])[i].toString()); + break; + case TEXT: + case STRING: + insertRecord.add( + new String(((Binary[]) values[j])[i].getValues(), TSFileConfig.STRING_CHARSET)); + break; + case BLOB: + String value = + BytesUtils.parseBlobByteArrayToString(((Binary[]) values[j])[i].getValues()) + .substring(2); + insertRecord.add(String.format("X'%s'", value)); + break; + case BOOLEAN: + insertRecord.add(String.valueOf(((boolean[]) values[j])[i])); + break; + } + } + insertRecords.add(insertRecord); + } + + return insertRecords; + } + + private Tablet generateTabletAndMeasurementSchema( + List> pairs, String deviceId) { + long[] timestamp = createTestDataForTimestamp(); + Object[] objects = new Object[pairs.size()]; + List measurementSchemas = new ArrayList<>(pairs.size()); + BitMap[] bitMaps = new BitMap[pairs.size()]; + for (int i = 0; i < bitMaps.length; i++) { + bitMaps[i] = new BitMap(generateDataSize); + } + List columnTypes = new ArrayList<>(pairs.size()); + for (int i = 0; i < objects.length; i++) { + MeasurementSchema schema = pairs.get(i).left; + measurementSchemas.add(schema); + columnTypes.add(ColumnCategory.FIELD); + switch (schema.getType()) { + case INT64: + objects[i] = createTestDataForInt64(); + break; + case INT32: + objects[i] = createTestDataForInt32(); + break; + case TIMESTAMP: + objects[i] = createTestDataForTimestamp(); + break; + case DOUBLE: + objects[i] = createTestDataForDouble(); + break; + case FLOAT: + objects[i] = createTestDataForFloat(); + break; + case DATE: + objects[i] = createTestDataForDate(); + break; + case STRING: + case BLOB: + case TEXT: + objects[i] = createTestDataForString(); + break; + case BOOLEAN: + objects[i] = createTestDataForBoolean(); + break; + } + } + return new Tablet(deviceId, measurementSchemas, timestamp, objects, bitMaps, generateDataSize); + } + + private List> generateMeasurementSchemas() { + TSDataType[] dataTypes = { + TSDataType.STRING, + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.TIMESTAMP, + TSDataType.BOOLEAN, + TSDataType.DATE, + TSDataType.DOUBLE, + TSDataType.FLOAT, + TSDataType.INT32, + TSDataType.INT64 + }; + List> pairs = new ArrayList<>(); + + for (TSDataType type : dataTypes) { + for (TSDataType dataType : dataTypes) { + String id = String.format("%s2%s", type.name(), dataType.name()); + pairs.add(new Pair<>(new MeasurementSchema(id, type), new MeasurementSchema(id, dataType))); + } + } + return pairs; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java new file mode 100644 index 0000000000000..e87bda0f361d4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java @@ -0,0 +1,612 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.RpcUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.nio.charset.StandardCharsets; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeTypeConversionIT extends AbstractPipeDualTreeModelManualIT { + + private static final int generateDataSize = 100; + + // Test for converting BOOLEAN to OtherType + @Test + public void testBooleanToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.BOOLEAN, TSDataType.DATE); + } + + // Test for converting INT32 to OtherType + @Test + public void testInt32ToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.INT32, TSDataType.DATE); + } + + // Test for converting INT64 to OtherType + @Test + public void testInt64ToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.INT64, TSDataType.DATE); + } + + // Test for converting FLOAT to OtherType + @Test + public void testFloatToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.FLOAT, TSDataType.DATE); + } + + // Test for converting DOUBLE to OtherType + @Test + public void testDoubleToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.DOUBLE, TSDataType.DATE); + } + + // Test for converting TEXT to OtherType + @Test + public void testTextToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.TEXT, TSDataType.DATE); + } + + // Test for converting TIMESTAMP to OtherType + @Test + public void testTimestampToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.TIMESTAMP, TSDataType.DATE); + } + + // Test for converting DATE to OtherType + @Test + public void testDateToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.DATE, TSDataType.TIMESTAMP); + } + + // Test for converting BLOB to OtherType + @Test + public void testBlobToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.STRING); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.TIMESTAMP); + executeAndVerifyTypeConversion(TSDataType.BLOB, TSDataType.DATE); + } + + // Test for converting STRING to OtherType + @Test + public void testStringToOtherTypeConversion() { + createDataPipe(); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.TEXT); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.BLOB); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.BOOLEAN); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.INT32); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.INT64); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.FLOAT); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.DOUBLE); + executeAndVerifyTypeConversion(TSDataType.STRING, TSDataType.TIMESTAMP); + } + + private void executeAndVerifyTypeConversion(TSDataType source, TSDataType target) { + List pairs = prepareTypeConversionTest(source, target); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + String.format("select status from root.test.%s2%s", source.name(), target.name()), + String.format("Time,root.test.%s2%s.status,", source.name(), target.name()), + createExpectedResultSet(pairs, source, target), + 600); + } + + private List prepareTypeConversionTest(TSDataType sourceType, TSDataType targetType) { + String sourceTypeName = sourceType.name(); + String targetTypeName = targetType.name(); + + createTimeSeries(sourceTypeName, targetTypeName, sourceTypeName, senderEnv); + createTimeSeries(sourceTypeName, targetTypeName, targetTypeName, receiverEnv); + + List pairs = createTestDataForType(sourceTypeName); + + executeDataInsertions(pairs, sourceType, targetType); + return pairs; + } + + private void createTimeSeries( + String sourceTypeName, String targetTypeName, String dataType, BaseEnv env) { + String timeSeriesCreationQuery = + String.format( + "create timeseries root.test.%s2%s.status with datatype=%s,encoding=PLAIN", + sourceTypeName, targetTypeName, dataType); + TestUtils.tryExecuteNonQueriesWithRetry( + env, Collections.singletonList(timeSeriesCreationQuery)); + } + + private void createDataPipe() { + String sql = + String.format( + "create pipe test" + + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='forced-log','realtime.enable'='true','history.enable'='false')" + + " with processor ('processor'='do-nothing-processor')" + + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='tablet')", + receiverEnv.getIP(), receiverEnv.getPort()); + TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList(sql)); + } + + private List createTestDataForType(String sourceType) { + switch (sourceType) { + case "BOOLEAN": + return createTestDataForBoolean(); + case "INT32": + return createTestDataForInt32(); + case "INT64": + return createTestDataForInt64(); + case "FLOAT": + return createTestDataForFloat(); + case "DOUBLE": + return createTestDataForDouble(); + case "TEXT": + return createTestDataForText(); + case "TIMESTAMP": + return createTestDataForTimestamp(); + case "DATE": + return createTestDataForDate(); + case "BLOB": + return createTestDataForBlob(); + case "STRING": + return createTestDataForString(); + default: + throw new UnsupportedOperationException("Unsupported data type: " + sourceType); + } + } + + private void executeDataInsertions( + List testData, TSDataType sourceType, TSDataType targetType) { + switch (sourceType) { + case STRING: + case TEXT: + TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + createInsertStatementsForString(testData, sourceType.name(), targetType.name())); + return; + case TIMESTAMP: + TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + createInsertStatementsForTimestamp(testData, sourceType.name(), targetType.name())); + return; + case DATE: + TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + createInsertStatementsForLocalDate(testData, sourceType.name(), targetType.name())); + return; + case BLOB: + TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + createInsertStatementsForBlob(testData, sourceType.name(), targetType.name())); + return; + default: + TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + createInsertStatementsForNumeric(testData, sourceType.name(), targetType.name())); + } + } + + private List createInsertStatementsForString( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into root.test.%s2%s(timestamp,status) values (%s,'%s');", + sourceType, + targetType, + pair.left, + new String(((Binary) (pair.right)).getValues(), StandardCharsets.UTF_8))); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForNumeric( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into root.test.%s2%s(timestamp,status) values (%s,%s);", + sourceType, targetType, pair.left, pair.right)); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForTimestamp( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into root.test.%s2%s(timestamp,status) values (%s,%s);", + sourceType, targetType, pair.left, pair.right)); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForLocalDate( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + executes.add( + String.format( + "insert into root.test.%s2%s(timestamp,status) values (%s,'%s');", + sourceType, targetType, pair.left, DateUtils.formatDate((Integer) pair.right))); + } + executes.add("flush"); + return executes; + } + + private List createInsertStatementsForBlob( + List testData, String sourceType, String targetType) { + List executes = new ArrayList<>(); + for (Pair pair : testData) { + String value = BytesUtils.parseBlobByteArrayToString(((Binary) pair.right).getValues()); + executes.add( + String.format( + "insert into root.test.%s2%s(timestamp,status) values (%s,X'%s');", + sourceType, targetType, pair.left, value.substring(2))); + } + executes.add("flush"); + return executes; + } + + private Set createExpectedResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + switch (targetType) { + case TIMESTAMP: + return generateTimestampResultSet(pairs, sourceType, targetType); + case DATE: + return generateLocalDateResultSet(pairs, sourceType, targetType); + case BLOB: + return generateBlobResultSet(pairs, sourceType, targetType); + case TEXT: + case STRING: + return generateStringResultSet(pairs, sourceType, targetType); + default: + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,", pair.left, ValueConverter.convert(sourceType, targetType, pair.right))); + } + return resultSet; + } + } + + private Set generateTimestampResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,", + pair.left, + RpcUtils.formatDatetime( + "default", + "ms", + (long) ValueConverter.convert(sourceType, targetType, pair.right), + ZoneOffset.UTC))); + } + return resultSet; + } + + private Set generateLocalDateResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,", + pair.left, + DateUtils.formatDate( + (Integer) ValueConverter.convert(sourceType, targetType, pair.right)))); + } + return resultSet; + } + + private Set generateBlobResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,", + pair.left, + BytesUtils.parseBlobByteArrayToString( + ((Binary) ValueConverter.convert(sourceType, targetType, pair.right)) + .getValues()))); + } + return resultSet; + } + + private Set generateStringResultSet( + List pairs, TSDataType sourceType, TSDataType targetType) { + HashSet resultSet = new HashSet<>(); + for (Pair pair : pairs) { + resultSet.add( + String.format( + "%s,%s,", + pair.left, + new String( + ((Binary) ValueConverter.convert(sourceType, targetType, pair.right)).getValues(), + StandardCharsets.UTF_8))); + } + return resultSet; + } + + private List createTestDataForBoolean() { + List pairs = new java.util.ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextBoolean())); + } + return pairs; + } + + private List createTestDataForInt32() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextInt())); + } + pairs.add(new Pair<>(generateDataSize + 1, -1)); + pairs.add(new Pair<>(generateDataSize + 2, -2)); + pairs.add(new Pair<>(generateDataSize + 3, -3)); + return pairs; + } + + private List createTestDataForInt64() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextLong())); + } + pairs.add(new Pair<>(generateDataSize + 1, -1L)); + pairs.add(new Pair<>(generateDataSize + 2, -2L)); + pairs.add(new Pair<>(generateDataSize + 3, -3L)); + return pairs; + } + + private List createTestDataForFloat() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextFloat())); + } + return pairs; + } + + private List createTestDataForDouble() { + List pairs = new ArrayList<>(); + Random random = new Random(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, random.nextDouble())); + } + return pairs; + } + + private List createTestDataForText() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Binary((String.valueOf(i)).getBytes(StandardCharsets.UTF_8)))); + } + pairs.add(new Pair(generateDataSize + 1, new Binary("Hello".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 2, new Binary("Hello World!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 3, new Binary("This is a test.".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 4, + new Binary("IoTDB Hello World!!!!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 5, + new Binary( + "IoTDB is an excellent time series database!!!!!!!!!" + .getBytes(StandardCharsets.UTF_8)))); + return pairs; + } + + private List createTestDataForTimestamp() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Date().getTime() + i)); + } + return pairs; + } + + private List createTestDataForDate() { + List pairs = new ArrayList<>(); + int year = 2023; + int month = 1; + int day = 1; + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, year * 10000 + (month * 100) + day)); + + // update + day++; + if (day > 28) { + day = 1; + month++; + if (month > 12) { + month = 1; + year++; + } + } + } + return pairs; + } + + private List createTestDataForBlob() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Binary((String.valueOf(i)).getBytes(StandardCharsets.UTF_8)))); + } + pairs.add(new Pair(generateDataSize + 1, new Binary("Hello".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 2, new Binary("Hello World!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 3, new Binary("This is a test.".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 4, + new Binary("IoTDB Hello World!!!!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 5, + new Binary( + "IoTDB is an excellent time series database!!!!!!!!!" + .getBytes(StandardCharsets.UTF_8)))); + return pairs; + } + + private List createTestDataForString() { + List pairs = new ArrayList<>(); + for (long i = 0; i < generateDataSize; i++) { + pairs.add(new Pair<>(i, new Binary((String.valueOf(i)).getBytes(StandardCharsets.UTF_8)))); + } + pairs.add(new Pair(generateDataSize + 1, new Binary("Hello".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 2, new Binary("Hello World!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 3, new Binary("This is a test.".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 4, + new Binary("IoTDB Hello World!!!!".getBytes(StandardCharsets.UTF_8)))); + pairs.add( + new Pair( + generateDataSize + 5, + new Binary( + "IoTDB is an excellent time series database!!!!!!!!!" + .getBytes(StandardCharsets.UTF_8)))); + return pairs; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java deleted file mode 100644 index ae81f1ffb20a3..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.manual; - -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.itbase.env.BaseEnv; - -import org.junit.After; -import org.junit.Before; - -abstract class AbstractPipeDualManualIT { - - protected BaseEnv senderEnv; - protected BaseEnv receiverEnv; - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @After - public final void tearDown() { - senderEnv.cleanClusterEnvironment(); - receiverEnv.cleanClusterEnvironment(); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java deleted file mode 100644 index b6fd01a8efe76..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.manual; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualManualIT { - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setDefaultSchemaRegionGroupNumPerDatabase(1) - .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(3) - .setDataReplicationFactor(2); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(3, 3); - } - - @Test - public void testTemplateInclusion() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create database root.ln", - "create database root.db", - "set ttl to root.ln 3600000", - "create user `thulab` 'passwd'", - "create role `admin`", - "grant role `admin` to `thulab`", - "grant read on root.** to role `admin`", - "create schema template t1 (temperature FLOAT encoding=RLE, status BOOLEAN encoding=PLAIN compression=SNAPPY)", - "set schema template t1 to root.ln.wf01", - "set schema template t1 to root.db.wf01", - "create timeseries using schema template on root.ln.wf01.wt01", - "create timeseries using schema template on root.db.wf01.wt01", - "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", - // Insert large timestamp to avoid deletion by ttl - "insert into root.ln.wf01.wt01(time, temperature, status) values (1800000000000, 23, true)"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "data, schema"); - extractorAttributes.put("extractor.inclusion.exclusion", "schema.timeseries.ordinary"); - extractorAttributes.put("extractor.path", "root.ln.**"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "list user", - ColumnHeaderConstant.USER + ",", - Collections.singleton("root,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "list role", ColumnHeaderConstant.ROLE + ",", Collections.emptySet()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "show databases", - "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionOrigin,TimePartitionInterval,", - // Receiver's SchemaReplicationFactor/DataReplicationFactor shall be 3/2 regardless of the - // sender - Collections.singleton("root.ln,3,2,0,604800000,")); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.**", - "Time,root.ln.wf01.wt01.temperature,root.ln.wf01.wt01.status,", - Collections.singleton("1800000000000,23.0,true,")); - - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "create timeseries using schema template on root.ln.wf01.wt02")) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton("4,")); - } - } - - @Test - public void testAuthInclusion() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create database root.ln", - "set ttl to root.ln 3600000", - "create user `thulab` 'passwd'", - "create role `admin`", - "grant role `admin` to `thulab`", - "grant read on root.** to role `admin`", - "create schema template t1 (temperature FLOAT encoding=RLE, status BOOLEAN encoding=PLAIN compression=SNAPPY)", - "set schema template t1 to root.ln.wf01", - "create timeseries using schema template on root.ln.wf01.wt01", - "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", - // Insert large timestamp to avoid deletion by ttl - "insert into root.ln.wf01.wt01(time, temperature, status) values (1800000000000, 23, true)"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "auth"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "list user of role `admin`", - ColumnHeaderConstant.USER + ",", - Collections.singleton("thulab,")); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "list privileges of role `admin`", - ColumnHeaderConstant.ROLE - + "," - + ColumnHeaderConstant.PATH - + "," - + ColumnHeaderConstant.PRIVILEGES - + "," - + ColumnHeaderConstant.GRANT_OPTION - + ",", - new HashSet<>( - Arrays.asList("admin,root.**,READ_DATA,false,", "admin,root.**,READ_SCHEMA,false,"))); - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "show databases", - "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionOrigin,TimePartitionInterval,", - Collections.emptySet()); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "select * from root.**", "Time", Collections.emptySet()); - - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "CREATE ROLE test")) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "list role", - ColumnHeaderConstant.ROLE + ",", - new HashSet<>(Arrays.asList("admin,", "test,"))); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java index 70bb4df4a93e8..4c5135764886d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java @@ -35,7 +35,7 @@ public void setUp() { env = MultiEnvFactory.getEnv(0); env.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); // 10 min, assert that the operations will not time out - env.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + env.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); env.initClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java index cbd9c08f4e5c2..78ad6b9bf3339 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java @@ -66,7 +66,8 @@ public void testAggregator() throws Exception { processorAttributes.put("output.database", "root.testdb"); processorAttributes.put( "output.measurements", "Avg1, peak1, rms1, var1, skew1, kurt1, ff1, cf1, pf1"); - processorAttributes.put("operators", "avg, peak, rms, var, skew, kurt, ff, cf, pf, cE"); + processorAttributes.put( + "operators", "avg, peak, rms, var, skew, kurt, ff, cf, pf, cE, max, min"); processorAttributes.put("sliding.seconds", "60"); connectorAttributes.put("sink", "write-back-sink"); @@ -115,7 +116,7 @@ public void testAggregator() throws Exception { env, "select count(*) from root.testdb.** group by level=1", "count(root.testdb.*.*.*.*),", - Collections.singleton("20,")); + Collections.singleton("24,")); // Test manually renamed timeSeries count TestUtils.assertDataEventuallyOnEnv( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java new file mode 100644 index 0000000000000..591881369ed41 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.single; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT1; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT1.class}) +public class IoTDBPipeOPCUAIT extends AbstractPipeSingleIT { + @Test + public void testOPCUASink() throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueryWithRetry( + env, "insert into root.db.d1(time, s1) values (1, 1)")) { + return; + } + + final Map connectorAttributes = new HashMap<>(); + connectorAttributes.put("sink", "opc-ua-sink"); + connectorAttributes.put("opcua.model", "client-server"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(Collections.emptyMap()) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); + + // Test reconstruction + connectorAttributes.put("password", "test"); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(Collections.emptyMap()) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + + // Test conflict + connectorAttributes.put("password", "conflict"); + Assert.assertEquals( + TSStatusCode.PIPE_ERROR.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(Collections.emptyMap()) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + } + } + + @Test + public void testOPCUASinkInTableModel() throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(env, "test", "test"); + if (!TestUtils.tryExecuteNonQueryWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + env, + "insert into test (s0, s1, s2) values (1, 1, 1)")) { + return; + } + + final Map connectorAttributes = new HashMap<>(); + connectorAttributes.put("sink", "opc-ua-sink"); + connectorAttributes.put("opcua.model", "client-server"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(Collections.singletonMap("capture.table", "true")) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); + + // Test reconstruction + connectorAttributes.put("password", "test"); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(Collections.emptyMap()) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + + // Test conflict + connectorAttributes.put("password", "conflict"); + Assert.assertEquals( + TSStatusCode.PIPE_ERROR.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(Collections.emptyMap()) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java new file mode 100644 index 0000000000000..64afeeb594ddf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.single; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT1; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT1.class}) +public class IoTDBPipePermissionIT extends AbstractPipeSingleIT { + @Test + public void testSinkPermission() { + if (!TestUtils.tryExecuteNonQueryWithRetry(env, "create user `thulab` 'passwd'")) { + return; + } + + // Shall fail if username is specified without password + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create pipe a2b ('user'='thulab', 'sink'='write-back-sink')"); + fail("When the 'user' or 'username' is specified, password must be specified too."); + } catch (final SQLException ignore) { + // Expected + } + + // Shall fail if password is wrong + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create pipe a2b ('user'='thulab', 'password'='hack', 'sink'='write-back-sink')"); + fail("Shall fail if password is wrong."); + } catch (final SQLException ignore) { + // Expected + } + + // Use current session, user is root + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create pipe a2b " + + "with source ('database'='test1') " + + "with processor('processor'='rename-database-processor', 'processor.new-db-name'='test') " + + "with sink ('sink'='write-back-sink')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Create pipe without user shall succeed if use the current session"); + } + + // Alter to another user, shall fail because of lack of password + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('username'='thulab')"); + fail("Alter pipe shall fail if only user is specified"); + } catch (final SQLException ignore) { + // Expected + } + + // Successfully alter + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('username'='thulab', 'password'='passwd')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Alter pipe shall not fail if user and password are specified"); + } + + TableModelUtils.createDataBaseAndTable(env, "test", "test1"); + TableModelUtils.createDataBaseAndTable(env, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(env, "test", "test"); + TableModelUtils.createDataBaseAndTable(env, "test1", "test"); + + // Write some data + if (!TableModelUtils.insertData("test1", "test", 0, 100, env)) { + return; + } + + // Filter this + if (!TestUtils.tryExecuteNonQueryWithRetry("test1", BaseEnv.TABLE_SQL_DIALECT, env, "flush")) { + return; + } + + TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, env); + + // Continue, ensure that it won't block + // Grant some privilege + if (!TestUtils.tryExecuteNonQueryWithRetry( + "test1", BaseEnv.TABLE_SQL_DIALECT, env, "grant INSERT on test.test1 to user thulab")) { + return; + } + if (!TableModelUtils.insertData("test1", "test1", 0, 100, env)) { + return; + } + TableModelUtils.assertCountData("test", "test1", 100, env); + + // Clear data, avoid resending + if (!TestUtils.tryExecuteNonQueryWithRetry( + "test", BaseEnv.TABLE_SQL_DIALECT, env, "drop database test1")) { + return; + } + + // Alter pipe, throw exception if no privileges + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify sink ('skipif'='')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + TableModelUtils.createDataBaseAndTable(env, "test", "test1"); + TableModelUtils.createDataBaseAndTable(env, "test1", "test1"); + + if (!TableModelUtils.insertData("test1", "test", 0, 100, env)) { + return; + } + + TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, env); + + // Grant some privilege + if (!TestUtils.tryExecuteNonQueryWithRetry( + "test", BaseEnv.TABLE_SQL_DIALECT, env, "grant INSERT on any to user thulab")) { + return; + } + + TableModelUtils.assertCountData("test", "test", 100, env); + } + + @Test + public void testSinkPermissionWithHistoricalDataAndTablePattern() { + TableModelUtils.createDataBaseAndTable(env, "test", "test1"); + TableModelUtils.createDataBaseAndTable(env, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(env, "test", "test"); + TableModelUtils.createDataBaseAndTable(env, "test1", "test"); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + env, + Arrays.asList( + "create user thulab 'passwd'", "grant INSERT on test.test1 to user thulab"))) { + return; + } + + // Write some data + if (!TableModelUtils.insertData("test1", "test", 0, 100, env)) { + return; + } + + if (!TableModelUtils.insertData("test1", "test1", 0, 100, env)) { + return; + } + + // Use current session, user is root + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create pipe a2b " + + "with source ('database'='test1', 'table'='test1') " + + "with processor('processor'='rename-database-processor', 'processor.new-db-name'='test') " + + "with sink ('sink'='write-back-sink', 'username'='thulab', 'password'='passwd')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Create pipe without user shall succeed if use the current session"); + } + + TableModelUtils.assertCountData("test", "test", 0, env); + TableModelUtils.assertCountData("test", "test1", 100, env); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAsofJoinTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAsofJoinTableIT.java new file mode 100644 index 0000000000000..eb48343a107f2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAsofJoinTableIT.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAsofJoinTableIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] sql = + new String[] { + "create database test", + "use test", + "create table table1(device string tag, value int32 field)", + "insert into table1(time,device,value) values(2020-01-01 00:00:01.000,'d1',1)", + "insert into table1(time,device,value) values(2020-01-01 00:00:03.000,'d1',3)", + "insert into table1(time,device,value) values(2020-01-01 00:00:05.000,'d1',5)", + "insert into table1(time,device,value) values(2020-01-01 00:00:08.000,'d2',8)", + "create table table2(device string tag, value int32 field)", + "insert into table2(time,device,value) values(2020-01-01 00:00:02.000,'d1',20)", + "insert into table2(time,device,value) values(2020-01-01 00:00:03.000,'d1',30)", + "insert into table2(time,device,value) values(2020-01-01 00:00:04.000,'d2',40)", + "insert into table2(time,device,value) values(2020-01-01 00:00:05.000,'d2',50)" + }; + String[] expectedHeader = + new String[] {"time1", "device1", "value1", "time2", "device2", "value2"}; + ; + String[] retArray; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setMaxTsBlockLineNumber(2) + .setMaxNumberOfPointsInPage(5); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sql) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void innerJoinTest() { + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,2020-01-01T00:00:03.000Z,d1,30,", + "2020-01-01T00:00:05.000Z,d1,5,2020-01-01T00:00:05.000Z,d2,50,", + "2020-01-01T00:00:08.000Z,d2,8,2020-01-01T00:00:05.000Z,d2,50," + }; + tableResultSetEqualTest( + "SELECT t1.time as time1, t1.device as device1, t1.value as value1, \n" + + " t2.time as time2, t2.device as device2, t2.value as value2 \n" + + "FROM \n" + + "table1 t1 ASOF INNER JOIN table2 t2\n" + + "ON\n" + + "t1.time>=t2.time\n" + + "order by time1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,2020-01-01T00:00:03.000Z,d1,30,", + "2020-01-01T00:00:05.000Z,d1,5,2020-01-01T00:00:05.000Z,d2,50,", + }; + tableResultSetEqualTest( + "SELECT t1.time as time1, t1.device as device1, t1.value as value1, \n" + + " t2.time as time2, t2.device as device2, t2.value as value2 \n" + + "FROM \n" + + "table1 t1 ASOF(tolerance 2s) INNER JOIN table2 t2\n" + + "ON\n" + + "t1.time>=t2.time\n" + + "order by time1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,2020-01-01T00:00:03.000Z,d1,30,", + "2020-01-01T00:00:05.000Z,d1,5,2020-01-01T00:00:03.000Z,d1,30,", + }; + tableResultSetEqualTest( + "SELECT t1.time as time1, t1.device as device1, t1.value as value1, \n" + + " t2.time as time2, t2.device as device2, t2.value as value2 \n" + + "FROM \n" + + "table1 t1 ASOF(tolerance 2s) INNER JOIN table2 t2\n" + + "ON\n" + + "t1.device=t2.device AND t1.time>=t2.time\n" + + "order by time1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void leftJoinTest() { + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,null,null,null,", + "2020-01-01T00:00:03.000Z,d1,3,2020-01-01T00:00:03.000Z,d1,30,", + "2020-01-01T00:00:05.000Z,d1,5,2020-01-01T00:00:05.000Z,d2,50,", + "2020-01-01T00:00:08.000Z,d2,8,2020-01-01T00:00:05.000Z,d2,50," + }; + tableResultSetEqualTest( + "SELECT t1.time as time1, t1.device as device1, t1.value as value1, \n" + + " t2.time as time2, t2.device as device2, t2.value as value2 \n" + + "FROM \n" + + "table1 t1 ASOF LEFT JOIN table2 t2\n" + + "ON\n" + + "t1.time>=t2.time\n" + + "order by time1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,null,null,null,", + "2020-01-01T00:00:03.000Z,d1,3,2020-01-01T00:00:03.000Z,d1,30,", + "2020-01-01T00:00:05.000Z,d1,5,2020-01-01T00:00:03.000Z,d1,30,", + "2020-01-01T00:00:08.000Z,d2,8,2020-01-01T00:00:05.000Z,d2,50," + }; + tableResultSetEqualTest( + "SELECT t1.time as time1, t1.device as device1, t1.value as value1, \n" + + " t2.time as time2, t2.device as device2, t2.value as value2 \n" + + "FROM \n" + + "table1 t1 ASOF LEFT JOIN table2 t2\n" + + "ON\n" + + "t1.device=t2.device AND t1.time>=t2.time\n" + + "order by time1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,2020-01-01T00:00:02.000Z,d1,20,", + "2020-01-01T00:00:03.000Z,d1,3,2020-01-01T00:00:04.000Z,d2,40,", + "2020-01-01T00:00:05.000Z,d1,5,null,null,null,", + "2020-01-01T00:00:08.000Z,d2,8,null,null,null," + }; + tableResultSetEqualTest( + "SELECT t1.time as time1, t1.device as device1, t1.value as value1, \n" + + " t2.time as time2, t2.device as device2, t2.value as value2 \n" + + "FROM \n" + + "table1 t1 ASOF LEFT JOIN table2 t2\n" + + "ON\n" + + "t1.time { + userStmt.execute( + String.format( + "load '%s' with ('database-level'='2', 'database-name'='test')", + tmpDir.getAbsolutePath())); + }); + } + + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("grant create on database test to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute( + String.format( + "load '%s' with ('database-level'='2', 'database-name'='test')", + tmpDir.getAbsolutePath())); + }); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("grant insert on any to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + userStmt.execute( + String.format( + "load '%s' with ('database-level'='2', 'database-name'='test')", + tmpDir.getAbsolutePath())); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java new file mode 100644 index 0000000000000..1470546f4f841 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +// TODO:fix the different type exception +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCaseWhenThenTableIT { + + private static final String DATABASE = "test"; + + private static final String[] expectedHeader = {"_col0"}; + + private static final String[] SQLs = + new String[] { + // normal cases + "CREATE DATABASE " + DATABASE, + "Use " + DATABASE, + "CREATE table table1 (device_id STRING TAG, s1 INT32 FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD)", + "CREATE table table2 (device_id STRING TAG, s3 FLOAT FIELD, s4 DOUBLE FIELD)", + "CREATE table table3 (device_id STRING TAG, s2 INT64 FIELD)", + "INSERT INTO table1(time, device_id, s1) values(100, 'd1', 0)", + "INSERT INTO table1(time, device_id, s1) values(200, 'd1', 11)", + "INSERT INTO table1(time, device_id, s1) values(300, 'd1', 22)", + "INSERT INTO table1(time, device_id, s1) values(400, 'd1', 33)", + "INSERT INTO table2(time, device_id, s3) values(100, 'd1', 0)", + "INSERT INTO table2(time, device_id, s3) values(200, 'd1', 11)", + "INSERT INTO table2(time, device_id, s3) values(300, 'd1', 22)", + "INSERT INTO table2(time, device_id, s3) values(400, 'd1', 33)", + "INSERT INTO table2(time, device_id, s4) values(100, 'd1', 44)", + "INSERT INTO table2(time, device_id, s4) values(200, 'd1', 55)", + "INSERT INTO table2(time, device_id, s4) values(300, 'd1', 66)", + "INSERT INTO table2(time, device_id, s4) values(400, 'd1', 77)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLs); + List moreSQLs = new ArrayList<>(); + moreSQLs.add("use " + DATABASE); + for (int i = 0; i < 100; i++) { + moreSQLs.add( + String.format("INSERT INTO table3(time,device_id,s2) values(%d, 'd1', %d)", i, i)); + } + prepareTableData(moreSQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testKind1Basic() { + String[] retArray = new String[] {"99,", "9999,", "9999,", "999,"}; + tableResultSetEqualTest( + "select case when s1=0 then 99 when s1>22 then 999 else 9999 end from table1", + expectedHeader, + retArray, + DATABASE); + + // without ELSE clause + retArray = new String[] {"99,", "null,", "null,", "999,"}; + tableResultSetEqualTest( + "select case when s1=0 then 99 when s1>22 then 999 end from table1", + expectedHeader, + retArray, + DATABASE); + } + + @Test + public void testKind2Basic() { + String sql = "select case s1 when 0 then 99 when 22 then 999 else 9999 end from table1"; + String[] retArray = new String[] {"99,", "9999,", "999,", "9999,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // without ELSE clause + sql = "select case s1 when 0 then 99 when 22 then 999 end from table1"; + retArray = new String[] {"99,", "null,", "999,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + @Test + public void testShortCircuitEvaluation() { + String[] retArray = new String[] {"0,", "11,", "22,", "33,"}; + tableResultSetEqualTest( + "select case when 1=0 then s1/0 when 1!=0 then s1 end from table1", + expectedHeader, + retArray, + DATABASE); + } + + @Test + public void testKind1InputTypeRestrict() { + // WHEN clause must return BOOLEAN + String sql = "select case when s1+1 then 20 else 22 end from table1"; + String msg = "701: CASE WHEN clause must evaluate to a BOOLEAN (actual: INT32)"; + tableAssertTestFail(sql, msg, DATABASE); + } + + @Test + public void testKind2InputTypeRestrict() { + // the expression in CASE clause must be able to be equated with the expression in WHEN clause + String sql = "select case s1 when '1' then 20 else 22 end from table1"; + String msg = "701: CASE operand type does not match WHEN clause operand type: INT32 vs STRING"; + tableAssertTestFail(sql, msg, DATABASE); + } + + @Test + public void testKind1OutputTypeRestrict() { + // BOOLEAN and other types cannot exist at the same time + String[] retArray = new String[] {"true,", "false,", "true,", "true,"}; + // success + tableResultSetEqualTest( + "select case when s1<=0 then true when s1=11 then false else true end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case when s1<=0 then true else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between BOOLEAN and INT32, all types (without duplicates): [BOOLEAN, INT32]", + DATABASE); + + // TEXT and other types cannot exist at the same time + retArray = new String[] {"good,", "bad,", "okok,", "okok,"}; + // success + tableResultSetEqualTest( + "select case when s1<=0 then 'good' when s1=11 then 'bad' else 'okok' end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case when s1<=0 then 'good' else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between STRING and INT32, all types (without duplicates): [STRING, INT32]", + DATABASE); + + // 4 numerical types(INT LONG FLOAT DOUBLE) can exist at the same time + // retArray = new String[] {"99.0,", "99.9,", "8.589934588E9,", "999.9999999999,"}; + // tableResultSetEqualTest( + // "select case when s1=0 then 99 when s1=11 then 99.9 when s1=22 then 8589934588 when + // s1=33 then 999.9999999999 else 10086 end from table1", + // expectedHeader, + // retArray, + // DATABASE); + } + + @Test + public void testKind2OutputTypeRestrict() { + // BOOLEAN and other types cannot exist at the same time + String[] retArray = + new String[] { + "true,", "false,", "true,", "true,", + }; + // success + tableResultSetEqualTest( + "select case s1 when 0 then true when 11 then false else true end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case s1 when 0 then true else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between BOOLEAN and INT32, all types (without duplicates): [BOOLEAN, INT32]", + DATABASE); + + // TEXT and other types cannot exist at the same time + retArray = new String[] {"good,", "bad,", "okok,", "okok,"}; + // success + tableResultSetEqualTest( + "select case s1 when 0 then 'good' when 11 then 'bad' else 'okok' end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case s1 when 0 then 'good' else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between STRING and INT32, all types (without duplicates): [STRING, INT32]", + DATABASE); + + // 4 numerical types(INT LONG FLOAT DOUBLE) can exist at the same time + // tableAssertTestFail( + // "select case s1 when 0 then 99 when 11 then 99.9 when 22 then 8589934588 when 33 then + // 999.9999999999 else 10086 end from table1", + // "All result types must be the same", + // DATABASE); + // retArray = new String[] {"99.0,", "99.9,", "8.589934588E9,", "999.9999999999,"}; + // tableResultSetEqualTest( + // "select case s1 when 0 then 99 when 11 then 99.9 when 22 then 8589934588 when 33 then + // 999.9999999999 else 10086 end from table1", + // expectedHeader, + // retArray, + // DATABASE); + } + + /** 100 branches */ + @Test + public void testKind1LargeNumberBranches() { + StringBuilder sqlBuilder = new StringBuilder(); + List retList = new ArrayList<>(); + sqlBuilder.append("select case "); + for (int i = 0; i < 100; i++) { + sqlBuilder.append(String.format("when s2=%d then s2*%d ", i, i * 100)); + retList.add(String.format("%d,", i * i * 100)); + } + sqlBuilder.append("end from table3"); + tableResultSetEqualTest( + sqlBuilder.toString(), expectedHeader, retList.toArray(new String[] {}), DATABASE); + } + + /** 100 branches */ + @Ignore + @Test + public void testKind2LargeNumberBranches() { + StringBuilder sqlBuilder = new StringBuilder(); + List retList = new ArrayList<>(); + sqlBuilder.append("select case s2 "); + for (int i = 0; i < 100; i++) { + sqlBuilder.append(String.format("when %d then s2*%d ", i, i * 100)); + retList.add(String.format("%d.0,", i * i * 100)); + } + sqlBuilder.append("end from table1"); + tableResultSetEqualTest( + sqlBuilder.toString(), expectedHeader, retList.toArray(new String[] {}), DATABASE); + } + + @Test + public void testKind1UsedInOtherOperation() { + String sql; + String[] retArray; + + // use in scalar operation + + // multiply + sql = "select 2 * case when s1=0 then 99 when s1=22.0 then 999 else 9999 end from table1"; + retArray = new String[] {"198,", "19998,", "1998,", "19998,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // add + sql = + "select " + + "case when s1=0 then 99 when s1=22.0 then 999 else 9999 end " + + "+" + + "case when s1=11 then 99 else 9999 end " + + "from table1"; + retArray = + new String[] { + "10098,", "10098,", "10998,", "19998,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // function + sql = "select diff(case when s1=0 then 99 when s1>22 then 999 else 9999 end) from table1"; + retArray = new String[] {"null,", "9900.0,", "0.0,", "-9000.0,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // use in aggregation operation + + // avg + // not support yet + + // max + // not support yet + + // avg × max_value + // not support yet + + // UDF is not allowed + } + + @Test + public void testKind2UsedInOtherOperation() { + String sql; + String[] retArray; + + // use in scalar operation + + // multiply + sql = "select 2 * case s1 when 0 then 99 when 22 then 999 else 9999 end from table1"; + retArray = new String[] {"198,", "19998,", "1998,", "19998,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // add + // sql = + // "select " + // + "case s1 when 0 then 99 when 22 then 999 else 9999 end " + // + "+" + // + "case s4 when 55.0 then 99 else 9999 end " + // + "from table1, table2"; + // + // retArray = + // new String[] { + // "10098,", "10098,", "10998,", "19998,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // function + sql = "select diff(case s1 when 0 then 99 when 22 then 999 else 9999 end) from table1"; + retArray = + new String[] { + "null,", "9900.0,", "-9000.0,", "9000.0,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // use in aggregation operation + + // avg + // not support yet + + // max_value + // not support yet + + // avg × max_value + // not support yet + + // UDF is not allowed + } + + @Test + public void testKind1UseOtherOperation() { + // WHEN-clause use scalar function + String sql = "select case when sin(s1)>=0 then '>0' else '<0' end from table1"; + String[] retArray = + new String[] { + ">0,", "<0,", "<0,", ">0,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // THEN-clause and ELSE-clause use scalar function + + // TODO: align by is not supported. + + // sql = + // "select case when s1<=11 then CAST(diff(s1) as TEXT) else CAST(s1-1 as TEXT) end from + // table1 align by device"; + // + // retArray = + // new String[] { + // "0,table1,null,", + // "1000000,table1,11.0,", + // "20000000,table1,21.0,", + // "210000000,table1,32.0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + @Test + public void testKind2UseOtherOperation() { + // CASE-clause use scalar function + String sql = + "select case round(sin(s1)) when 0 then '=0' when -1 then '<0' else '>0' end from table1"; + + tableAssertTestFail( + sql, + "701: CASE operand type does not match WHEN clause operand type: DOUBLE vs INT32", + DATABASE); + // String[] retArray = + // new String[] { + // "=0,", "<0,", ">0,", ">0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // WHEN-clause use scalar function + sql = "select case 0 when sin(s1) then '=0' else '!=0' end from table1"; + tableAssertTestFail( + sql, + "701: CASE operand type does not match WHEN clause operand type: INT32 vs DOUBLE", + DATABASE); + // retArray = + // new String[] { + // "=0,", "!=0,", "!=0,", "!=0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // THEN-clause and ELSE-clause use scalar function + // sql = + // "select case s1 when 11 then CAST(diff(s1) as TEXT) else CAST(s1-1 as TEXT) end from + // table1 align by device"; + // + // retArray = + // new String[] { + // "table1,-1.0,", "table1,11.0,", "table1,21.0,", "table1,32.0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // UDF is not allowed + // sql = "select case s1 when 0 then change_points(s1) end from table1"; + // String msg = "301: CASE expression cannot be used with non-mappable UDF"; + // tableAssertTestFail(sql, msg, DATABASE); + } + + @Test + @Ignore + public void testKind1Wildcard() { + String sql = "select case when *=* then * else * end from root.sg.d2"; + String[] expectedHeaders = new String[16]; + for (int i = 0; i < expectedHeaders.length; i++) { + expectedHeaders[i] = "_col" + i; + } + String[] retArray = { + "0.0,0.0,44.0,44.0,0.0,44.0,0.0,44.0,0.0,44.0,0.0,44.0,0.0,0.0,44.0,44.0,", + "11.0,11.0,55.0,55.0,11.0,55.0,11.0,55.0,11.0,55.0,11.0,55.0,11.0,11.0,55.0,55.0,", + "22.0,22.0,66.0,66.0,22.0,66.0,22.0,66.0,22.0,66.0,22.0,66.0,22.0,22.0,66.0,66.0,", + "33.0,33.0,77.0,77.0,33.0,77.0,33.0,77.0,33.0,77.0,33.0,77.0,33.0,33.0,77.0,77.0,", + }; + tableResultSetEqualTest(sql, expectedHeaders, retArray, DATABASE); + } + + @Test + @Ignore + public void testKind2Wildcard() { + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("select "); + String[] caseSQL = + new String[] { + "CASE WHEN s3 = s3 THEN s3 ELSE s3 END,", + "CASE WHEN s3 = s3 THEN s3 ELSE s4 END,", + "CASE WHEN s3 = s3 THEN s4 ELSE s3 END,", + "CASE WHEN s3 = s3 THEN s4 ELSE s4 END,", + "CASE WHEN s3 = s4 THEN s3 ELSE s3 END,", + "CASE WHEN s3 = s4 THEN s3 ELSE s4 END,", + "CASE WHEN s3 = s4 THEN s4 ELSE s3 END,", + "CASE WHEN s3 = s4 THEN s4 ELSE s4 END,", + "CASE WHEN s4 = s3 THEN s3 ELSE s3 END,", + "CASE WHEN s4 = s3 THEN s3 ELSE s4 END,", + "CASE WHEN s4 = s3 THEN s4 ELSE s3 END,", + "CASE WHEN s4 = s3 THEN s4 ELSE s4 END,", + "CASE WHEN s4 = s4 THEN s3 ELSE s3 END,", + "CASE WHEN s4 = s4 THEN s3 ELSE s4 END,", + "CASE WHEN s4 = s4 THEN s4 ELSE s3 END,", + "CASE WHEN s4 = s4 THEN s4 ELSE s4 END", + }; + for (String string : caseSQL) { + sqlBuilder.append(string); + } + sqlBuilder.append(" from table2"); + + String[] expectedHeaders = new String[16]; + for (int i = 0; i < 16; i++) { + expectedHeaders[i] = "_col" + i; + } + + String[] retArray = { + "0,0.0,0.0,44.0,44.0,0.0,44.0,0.0,44.0,0.0,44.0,0.0,44.0,0.0,0.0,44.0,44.0,", + "1000000,11.0,11.0,55.0,55.0,11.0,55.0,11.0,55.0,11.0,55.0,11.0,55.0,11.0,11.0,55.0,55.0,", + "20000000,22.0,22.0,66.0,66.0,22.0,66.0,22.0,66.0,22.0,66.0,22.0,66.0,22.0,22.0,66.0,66.0,", + "210000000,33.0,33.0,77.0,77.0,33.0,77.0,33.0,77.0,33.0,77.0,33.0,77.0,33.0,33.0,77.0,77.0,", + }; + tableResultSetEqualTest(sqlBuilder.toString(), expectedHeaders, retArray, DATABASE); + } + + @Ignore + @Test + public void testKind1AlignedByDevice() { + // from different devices, result should be empty + String sql = "select case when s1<=11 then s3 else s4 end from table1, table2 align by device"; + String[] retArray = {}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // from same device + sql = "select case when s3<=11 then s3 else s4 end from table1, root.sg.d2 align by device"; + retArray = + new String[] { + "root.sg.d2,0.0,", "root.sg.d2,11.0,", "root.sg.d2,66.0,", "root.sg.d2,77.0,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // from same device, two result column + sql = + "select " + + "case when s1<=11 then s1 else s1*2 end, " + + "case when s3<=11 then s3 else s4 end " + + "from table1, table2 align by device"; + + retArray = + new String[] { + "table1,0.0,null,", + "table1,11.0,null,", + "table1,44.0,null,", + "table1,66.0,null,", + "root.sg.d2,null,0.0,", + "root.sg.d2,null,11.0,", + "root.sg.d2,null,66.0,", + "root.sg.d2,null,77.0,", + }; + tableResultSetEqualTest(sql, new String[] {"_col0", "_col1"}, retArray, DATABASE); + } + + @Ignore + @Test + public void testKind2AlignedByDevice() { + // from different devices, result should be empty + String sql = "select case s1 when 11 then s3 else s4 end from table1, table2 align by device"; + String[] retArray = {}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // from same device + sql = "select case s3 when 11 then s3 else s4 end from table1, root.sg.d2 align by device"; + retArray = + new String[] { + "table2,44.0,", "table2,11.0,", "table2,66.0,", "table2,77.0,", + }; + resultSetEqualTest(sql, expectedHeader, retArray); + + // from same device, two result column + sql = + "select " + + "case s1 when 11 then s1 else s1*2 end, " + + "case s3 when 11 then s3 else s4 end " + + "from table1, root.sg.d2 align by device"; + retArray = + new String[] { + "0,table1,0.0,null,", + "1000000,table1,11.0,null,", + "20000000,table1,44.0,null,", + "210000000,table1,66.0,null,", + "0,root.sg.d2,null,44.0,", + "1000000,root.sg.d2,null,11.0,", + "20000000,root.sg.d2,null,66.0,", + "210000000,root.sg.d2,null,77.0,", + }; + resultSetEqualTest(sql, expectedHeader, retArray); + } + + @Ignore + @Test + public void testKind1MultipleTimeseries() { + // time stamp is aligned + String sql = "select s1*s1, case when s1<=11 then s3 else s4 end from table1, table2"; + String[] retArray = + new String[] { + "0.0,0.0,", "121.0,11.0,", "484.0,66.0,", "1089.0,77.0,", + }; + tableResultSetEqualTest(sql, new String[] {"_col0,_col1"}, retArray, DATABASE); + + // time stamp is not aligned + sql = + "select " + + "case when s2%2==1 then s2 else s2/2 end, " + + "case when s1<=11 then s3 else s4 end " + + "from table1, table2 limit 5 offset 98"; + + retArray = + new String[] { + "98,49.0,null,", + "99,99.0,null,", + "1000000,null,11.0,", + "20000000,null,66.0,", + "210000000,null,77.0,", + }; + tableResultSetEqualTest(sql, new String[] {"_col0", "_col1"}, retArray, DATABASE); + } + + @Ignore + @Test + public void testKind2MultipleTimeseries() { + // time stamp is aligned + String sql = "select s1*s1, case s1 when 11 then s3 else s4 end from table1, table2"; + String[] retArray = + new String[] { + "0.0,44.0,", "121.0,11.0,", "484.0,66.0,", "1089.0,77.0,", + }; + tableResultSetEqualTest(sql, new String[] {"_col0", "_col1"}, retArray, DATABASE); + + // time stamp is not aligned + sql = + "select " + + "case s2%2 when 1 then s2 else s2/2 end, " + + "case s3 when 11 then s3 else s4 end " + + "from table1, root.sg.d2 limit 5 offset 98"; + retArray = + new String[] { + "49.0,null,", "99.0,null,", "null,11.0,", "null,66.0,", "null,77.0,", + }; + resultSetEqualTest(sql, new String[] {"_col0", "_col1"}, retArray); + } + + @Test + public void testKind1UseInWhereClause() { + String sql = + "select s4 from table2 where case when s3=0 then s4>44 when s3=22 then s4>0 when time>300 then true end"; + String[] retArray = new String[] {"66.0,", "77.0,"}; + tableResultSetEqualTest(sql, new String[] {"s4"}, retArray, DATABASE); + + sql = + "select case when s3=0 then s4>44 when s3=22 then s4>0 when time>300 then true end from table2"; + retArray = + new String[] { + "false,", "null,", "true,", "true,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + @Ignore + @Test + public void testKind2UseInWhereClause() { + String sql = "select s4 from table2 where case s3 when 0 then s4>44 when 22 then s4>0 end"; + String[] retArray = new String[] {"66.0,"}; + tableResultSetEqualTest(sql, new String[] {"s4"}, retArray, DATABASE); + + // CASE time + sql = + "select s4 from table2 where case time when 0 then false when 300 then true when 200 then true end"; + retArray = new String[] {"55.0,", "66.0,"}; + tableResultSetEqualTest(sql, new String[] {"s4"}, retArray, DATABASE); + } + + @Test + public void testKind1CaseInCase() { + String sql = + "select case when s1=0 OR s1=22 then cast(case when s1=0 then 99 when s1>22 then 999 end as STRING) else 'xxx' end from table1"; + String[] retArray = + new String[] { + "99,", "xxx,", "null,", "xxx,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + @Test + public void testKind2CaseInCase() { + String sql = + "select case s1 when 0 then cast(case when s1=0 then 99 when s1>22 then 999 end as STRING) when 22 then cast(case when s1=0 then 99 when s1>22 then 999 end as STRING) else 'xxx' end from table1"; + String[] retArray = + new String[] { + "99,", "xxx,", "null,", "xxx,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + @Test + public void testKind1Logic() { + String sql = + "select case when s3 >= 0 and s3 < 20 and s4 >= 50 and s4 < 60 then 'just so so~~~' when s3 >= 20 and s3 < 40 and s4 >= 70 and s4 < 80 then 'very well~~~' end from table2"; + String[] retArray = new String[] {"null,", "just so so~~~,", "null,", "very well~~~,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java new file mode 100644 index 0000000000000..fb98c1620260c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -0,0 +1,2073 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ManualIT; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.itbase.exception.ParallelRequestTimeoutException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.TimeRange; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Random; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDeletionTableIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDeletionTableIT.class); + private static final String[] creationSqls = + new String[] { + "CREATE DATABASE IF NOT EXISTS test", + "USE test", + "CREATE TABLE IF NOT EXISTS vehicle0(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + }; + + private final String insertTemplate = + "INSERT INTO test.vehicle%d(time, deviceId, s0,s1,s2,s3,s4" + + ") VALUES(%d,'d%d',%d,%d,%f,%s,%b)"; + + @BeforeClass + public static void setUpClass() { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setPartitionInterval(1000) + .setMemtableSizeThreshold(10000); + // Adjust MemTable threshold size to make it flush automatically + EnvFactory.getEnv().getConfig().getDataNodeConfig().setCompactionScheduleInterval(5000); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Before + public void setUp() { + prepareDatabase(); + } + + @After + public void tearDown() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS test"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDownClass() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDeleteTimeWithSort1() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table t1(device_id string tag, s0 int32 field, s1 int32 field, s3 int32 field)"); + statement.execute( + "insert into t1(time, device_id, s0, s1, s3) values (10, 'device_1', 100, 100, 200)"); + statement.execute( + "insert into t1(time, device_id, s0, s1, s3) values (150, 'device_1', 100, 100, 200)"); + statement.execute( + "insert into t1(time, device_id, s0, s1, s3) values (202, 'device_1', 100, 100, 200)"); + statement.execute("delete from t1 where time <= 200 and device_id='device_1'"); + statement.execute( + "insert into t1(time, device_id, s0, s1, s3) values (10, 'device_1', 100, 100, 200)"); + statement.execute( + "insert into t1(time, device_id, s0, s1, s3) values (10, 'device_1', 100, 100, 200)"); + statement.execute("delete from t1 where time <= 200 and device_id='device_1'"); + + ResultSet resultSet = statement.executeQuery("select * from t1"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + count = 0; + + statement.execute( + "insert into t1(time, device_id, s0, s1, s3) values (10, 'device_1', 100, 100, 200)"); + statement.execute("delete from t1 where time <= 200 and device_id='device_1'"); + resultSet = statement.executeQuery("select * from t1"); + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } + } + + @Test + public void testDeleteTimeWithSort2() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table t2(device_id string tag, s0 int32 field, s1 int32 field, s3 int32 field)"); + statement.execute( + "insert into t2(time, device_id, s0, s1, s3) values (10, 'device_1', 100, 100, 200)"); + statement.execute( + "insert into t2(time, device_id, s0, s1, s3) values (150, 'device_1', 100, 100, 200)"); + statement.execute( + "insert into t2(time, device_id, s0, s1, s3) values (202, 'device_1', 100, 100, 200)"); + statement.execute("delete from t2 where time <= 200 and device_id='device_1'"); + + ResultSet resultSet = statement.executeQuery("select * from t2"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + count = 0; + + statement.execute( + "insert into t2(time, device_id, s0, s1, s3) values (10, 'device_1', 100, 100, 200)"); + statement.execute("delete from t2 where time <= 200 and device_id='device_1'"); + + resultSet = statement.executeQuery("select * from t2"); + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } + } + + /** Should delete this case after the deletion value filter feature be implemented */ + @Test + public void testUnsupportedValueFilter() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "CREATE TABLE vehicle1(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD, attr1 ATTRIBUTE)"); + + statement.execute("insert into vehicle1(time, deviceId, s0) values (10, 'd0', 310)"); + statement.execute("insert into vehicle1(time, deviceId, s3) values (10, 'd0','text')"); + statement.execute("insert into vehicle1(time, deviceId, s4) values (10, 'd0',true)"); + + try { + statement.execute("DELETE FROM vehicle1 WHERE s0 <= 300 AND s0 > 0"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: The column 's0' does not exist or is not a tag column", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE s1 = 'text'"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: The column 's1' does not exist or is not a tag column", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE attr1 = 'text'"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals( + "701: The column 'attr1' does not exist or is not a tag column", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE s3 = 'text'"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: The column 's3' does not exist or is not a tag column", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE s4 != true"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: The column 's4' does not exist or is not a tag column", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE time < 10 and deviceId > 'd0'"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: The operator of tag predicate must be '=' for 'd0'", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE time < 10 and deviceId is not null"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals( + "701: Unsupported expression: (deviceId IS NOT NULL) in ((time < 10) AND (deviceId IS NOT NULL))", + e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE time < 10 and deviceId = null"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals( + "701: The right hand value of tag predicate cannot be null with '=' operator, please use 'IS NULL' instead", + e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicle1 WHERE true"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: Unsupported expression: true in true", e.getMessage()); + } + + try { + statement.execute("DELETE FROM vehicleNonExist"); + fail("should not reach here!"); + } catch (SQLException e) { + assertEquals("701: Table vehiclenonexist not found", e.getMessage()); + } + + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle1")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(1, cnt); + } + + try (ResultSet set = statement.executeQuery("SELECT s3 FROM vehicle1")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(1, cnt); + } + + try (ResultSet set = statement.executeQuery("SELECT s4 FROM vehicle1")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(1, cnt); + } + } + } + + @Test + public void test() throws SQLException { + int testId = 2; + prepareData(testId, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + // init [1, 400] + + // remain [151, 400] + statement.execute("DELETE FROM vehicle" + testId + " WHERE time <= 150"); + try (ResultSet set = statement.executeQuery("SELECT * FROM vehicle" + testId)) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(250, cnt); + } + + // remain [301, 400] + statement.execute("DELETE FROM vehicle" + testId + " WHERE time <= 300"); + + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle" + testId)) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(100, cnt); + } + + // remain [351, 400] + statement.execute("DELETE FROM vehicle" + testId + " WHERE time <= 350"); + + try (ResultSet set = statement.executeQuery("SELECT s1,s2,s3 FROM vehicle" + testId)) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(50, cnt); + } + + // remain [361, 380] + statement.execute("DELETE FROM vehicle" + testId + " WHERE time <= 360 or time > 380"); + try (ResultSet set = statement.executeQuery("SELECT s1,s2,s3 FROM vehicle" + testId)) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(20, cnt); + } + } + cleanData(testId); + } + + @Test + public void testDelAfterFlush() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE ln3"); + statement.execute("use ln3"); + statement.execute( + String.format( + "CREATE TABLE vehicle3(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)")); + + statement.execute( + "INSERT INTO vehicle3(time, deviceId, s4) " + "values(1509465600000, 'd0', true)"); + statement.execute("INSERT INTO vehicle3(time, deviceId, s4) VALUES(NOW(), 'd0', false)"); + + statement.execute("delete from vehicle3 where time <= NOW()"); + statement.execute("flush"); + statement.execute("delete from vehicle3 where time <= NOW()"); + + try (ResultSet resultSet = statement.executeQuery("select s4 from vehicle3")) { + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testRangeDelete() throws SQLException { + prepareData(4, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + // [1, 400] -> [1, 299] + statement.execute("DELETE FROM vehicle4 WHERE time >= 300"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle4")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(299, cnt); + } + + // [1, 299] -> [151, 299] + statement.execute("DELETE FROM vehicle4 WHERE time <= 150"); + try (ResultSet set = statement.executeQuery("SELECT s1 FROM vehicle4")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(149, cnt); + } + + // [151, 299] -> [251, 299] + statement.execute("DELETE FROM vehicle4 WHERE time > 50 and time <= 250"); + try (ResultSet set = statement.executeQuery("SELECT * FROM vehicle4")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(49, cnt); + } + } + cleanData(4); + } + + @Test + public void testSuccessfullyInvalidateCache() throws SQLException { + prepareData(4, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.executeQuery( + "SELECT last(time), last_by(s0,time), last_by(s1,time), last_by(s2,time), last_by(s3,time), last_by(s4,time) FROM vehicle4 where deviceId = 'd0'"); + + // [1, 400] -> [1, 299] + statement.execute("DELETE FROM vehicle4 WHERE time >= 300"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle4")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(299, cnt); + } + } + cleanData(4); + } + + @Test + public void testFullDeleteWithoutWhereClause() throws SQLException { + prepareData(5, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute("DELETE FROM vehicle5"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle5")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(0, cnt); + } + cleanData(5); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testDeleteWithSpecificDevice() throws SQLException { + prepareData(6, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + statement.execute( + "DELETE FROM vehicle6 WHERE time <= 300 and time > 150 and deviceId = 'd0'"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle6")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(250, cnt); + } + + // Invalid deletion, d1 not exists + statement.execute("DELETE FROM vehicle6 WHERE time <= 200 and deviceId = 'd1'"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle6")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(250, cnt); + } + } + cleanData(6); + } + + @Test + public void testDelFlushingMemTable() throws SQLException { + int testNum = 7; + int deviceId = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + for (int i = 1; i <= 10000; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, deviceId, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + + statement.execute("DELETE FROM vehicle7 WHERE time > 1500 and time <= 9000"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle7")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(2500, cnt); + } + cleanData(testNum); + } + } + + @Test + public void testDelMultipleFlushingMemTable() throws SQLException { + int testNum = 8; + int deviceId = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + for (int i = 1; i <= 1000; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, deviceId, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + + statement.execute("DELETE FROM vehicle8 WHERE time > 150 and time <= 300"); + statement.execute("DELETE FROM vehicle8 WHERE time > 300 and time <= 400"); + for (int i = 1001; i <= 2000; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, deviceId, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + + statement.execute("DELETE FROM vehicle8 WHERE time > 500 and time <= 800"); + statement.execute("DELETE FROM vehicle8 WHERE time > 900 and time <= 1100"); + statement.execute("DELETE FROM vehicle8 WHERE time > 1500 and time <= 1650"); + statement.execute("flush"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle8")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(1100, cnt); + } + cleanData(testNum); + } + } + + @Test + public void testDeleteAll() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + 9)); + + statement.execute("insert into vehicle9(time, deviceId, s2) values(9,'d0',9.8)"); + statement.execute("insert into vehicle9(time, deviceId, s2) values(11, 'd0', 4.5)"); + + try (ResultSet resultSet = statement.executeQuery("select * from vehicle9")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(2, cnt); + } + + statement.execute("delete from vehicle9"); + + try (ResultSet resultSet = statement.executeQuery("select * from vehicle9")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Test + public void testDeleteDataFromEmptyTable() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + String.format( + "CREATE TABLE vehicle%d_1(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + 10)); + statement.execute( + String.format( + "CREATE TABLE vehicle%d_2(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + 10)); + + statement.execute( + "INSERT INTO vehicle10_2(Time, deviceId, s4) VALUES (2022-10-11 10:20:50,'d0', true),(2022-10-11 10:20:51,'d0',true)"); + statement.execute("DELETE FROM vehicle10_1 WHERE time >2022-10-11 10:20:50"); + + try (ResultSet resultSet = statement.executeQuery("select * from vehicle10_2")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(2, cnt); + } + } + } + + @Test + public void testDelSeriesWithSpecialSymbol() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + 11)); + + statement.execute("INSERT INTO vehicle11(time, deviceId, s4) VALUES(300, 'device,1', true)"); + statement.execute("INSERT INTO vehicle11(time, deviceId, s4) VALUES(500, 'device,2', false)"); + + try (ResultSet resultSet = statement.executeQuery("select * from vehicle11")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(2, cnt); + } + + statement.execute("DELETE FROM vehicle11 WHERE time <= 400 and deviceId = 'device,1'"); + + try (ResultSet resultSet = statement.executeQuery("select * from vehicle11")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + statement.execute("DELETE FROM vehicle11 WHERE deviceId = 'device,2'"); + + try (ResultSet resultSet = statement.executeQuery("select * from vehicle11")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Test + public void testDeleteTable() throws SQLException { + int testNum = 12; + prepareData(testNum, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + statement.execute("DROP TABLE vehicle" + testNum); + + try (ResultSet ignored = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { + fail("Exception expected"); + } catch (SQLException e) { + assertEquals("550: Table 'test.vehicle12' does not exist.", e.getMessage()); + } + + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + try (ResultSet set = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { + assertFalse(set.next()); + } + + prepareData(testNum, 1); + + statement.execute("DELETE FROM vehicle" + testNum + " WHERE time <= 150"); + + try (ResultSet set = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(250, cnt); + } + } + cleanData(testNum); + } + + @Test + public void testSingleDeviceDeletionMultiExecution() throws SQLException { + int testNum = 13; + prepareData(testNum, 5); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + // init d0[1, 400] d1[1, 400] d2[1, 400] d3[1, 400] d4[1, 400] + + // remain d1[10, 400] d2[10, 400] d3[10, 400] d4[10, 400] + statement.execute("DELETE FROM vehicle" + testNum + " WHERE time < 10 or deviceId = 'd0'"); + int[] expectedPointNumOfDevice = new int[] {0, 391, 391, 391, 391}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d1[50, 400] d2[10, 400] d3[10, 400] d4[10, 400] + statement.execute("DELETE FROM vehicle" + testNum + " WHERE time < 50 and deviceId = 'd1'"); + expectedPointNumOfDevice = new int[] {0, 351, 391, 391, 391}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d1[50, 400] d2[101, 400] d3[10, 400] d4[10, 400] + statement.execute( + "DELETE FROM vehicle" + testNum + " WHERE time <= 100 and deviceId = 'd2'"); + expectedPointNumOfDevice = new int[] {0, 351, 300, 391, 391}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d1[50, 400] d2[101, 400] d3[301, 400] d4[10, 400] + statement.execute( + "DELETE FROM vehicle" + testNum + " WHERE time <= 300 and deviceId = 'd3'"); + expectedPointNumOfDevice = new int[] {0, 351, 300, 100, 391}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d1[50, 400] d2[101, 400] d3[301, 400] d4[10, 100] + statement.execute("DELETE FROM vehicle" + testNum + " WHERE time > 100 and deviceId = 'd4'"); + expectedPointNumOfDevice = new int[] {0, 351, 300, 100, 91}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + } + cleanData(testNum); + } + + private void checkDevicePoint(int[] expectedPointNumOfDevice, Statement statement, int testNum) + throws SQLException { + for (int i = 0; i < expectedPointNumOfDevice.length; i++) { + try (ResultSet set = + statement.executeQuery( + "SELECT * FROM vehicle" + testNum + " where deviceId = 'd" + i + "'")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(expectedPointNumOfDevice[i], cnt); + } + } + } + + @Test + public void testDeviceIdWithNull() throws SQLException { + int testNum = 14; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table t" + testNum + " (tag1 string tag, tag2 string tag, s1 int32 field)"); + // tag1 is null for this record + statement.execute("insert into t" + testNum + " (time, tag2, s1) values (1, '1', 1)"); + statement.execute("insert into t" + testNum + " (time, tag2, s1) values (2, '', 2)"); + statement.execute("insert into t" + testNum + " (time, tag2, s1) values (3, NULL, 3)"); + statement.execute("flush"); + + statement.execute("delete from t" + testNum + " where tag1 is NULL and time <= 1"); + try (ResultSet set = statement.executeQuery("SELECT * FROM t" + testNum + " order by time")) { + assertTrue(set.next()); + assertEquals(2, set.getLong("time")); + assertTrue(set.next()); + assertEquals(3, set.getLong("time")); + assertFalse(set.next()); + } + + statement.execute("delete from t" + testNum + " where tag2 is NULL"); + try (ResultSet set = statement.executeQuery("SELECT * FROM t" + testNum + " order by time")) { + assertTrue(set.next()); + assertEquals(2, set.getLong("time")); + assertFalse(set.next()); + } + + statement.execute("delete from t" + testNum); + try (ResultSet set = statement.executeQuery("SELECT * FROM t" + testNum + " order by time")) { + assertFalse(set.next()); + } + + statement.execute("drop table t" + testNum); + } + } + + @Test + public void testEmptyString() throws SQLException { + int testNum = 15; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table t" + testNum + " (tag1 string tag, tag2 string tag, s1 int32 field)"); + // tag1 is null for this record + statement.execute("insert into t" + testNum + " (time, tag2, s1) values (1, '1', 1)"); + statement.execute("insert into t" + testNum + " (time, tag2, s1) values (2, '', 2)"); + statement.execute("insert into t" + testNum + " (time, tag2, s1) values (3, NULL, 3)"); + statement.execute("flush"); + + statement.execute("delete from t" + testNum + " where tag2 = ''"); + try (ResultSet set = statement.executeQuery("SELECT * FROM t" + testNum + " order by time")) { + assertTrue(set.next()); + assertEquals(1, set.getLong("time")); + assertTrue(set.next()); + assertEquals(3, set.getLong("time")); + assertFalse(set.next()); + } + + statement.execute("drop table t" + testNum); + } + } + + @Test + public void testIllegalRange() throws SQLException { + int testNum = 16; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table t" + testNum + " (tag1 string tag, tag2 string tag, s1 int32 field)"); + + try { + statement.execute("delete from t" + testNum + " where time > 10 and time <= 1"); + fail("Exception expected"); + } catch (SQLException e) { + assertEquals("701: Start time 11 is greater than end time 1", e.getMessage()); + } + } + } + + @Test + public void testMultiDevicePartialDeletionMultiExecution() throws SQLException { + int testNum = 17; + prepareData(testNum, 5); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + // init d0[1, 400] d1[1, 400] d2[1, 400] d3[1, 400] d4[1, 400] + + // remain d0[10, 400] d1[10, 400] d2[1, 400] d3[1, 400] d4[1, 400] + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE time < 10 and (deviceId = 'd0' or deviceId = 'd1')"); + int[] expectedPointNumOfDevice = new int[] {391, 391, 400, 400, 400}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d0[10, 400] d1[50, 400] d2[50, 400] d3[50, 400] d4[1, 400] + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE time < 50 and (deviceId = 'd1' or deviceId = 'd2' or deviceId = 'd3')"); + expectedPointNumOfDevice = new int[] {391, 351, 351, 351, 400}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d0[101, 400] d1[50, 400] d2[101, 400] d3[101, 400] d4[101, 400] + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE time <= 100 and (deviceId = 'd2' or deviceId = 'd3' or deviceId = 'd4' or deviceId = 'd0')"); + expectedPointNumOfDevice = new int[] {300, 351, 300, 300, 300}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d0[101, 150] d1[50, 150] d2[101, 150] d3[101, 150] d4[101, 150] + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE time > 150 and (deviceId = 'd2' or deviceId = 'd3' or deviceId = 'd4' or deviceId = 'd0' or deviceId = 'd1')"); + expectedPointNumOfDevice = new int[] {50, 101, 50, 50, 50}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + } + cleanData(testNum); + } + + @Test + public void testMultiDeviceFullDeletionMultiExecution() throws SQLException { + int testNum = 18; + prepareData(testNum, 5); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + + // init d0[1, 400] d1[1, 400] d2[1, 400] d3[1, 400] d4[1, 400] + + // remain d2[1, 400] d3[1, 400] d4[1, 400] + statement.execute( + "DELETE FROM vehicle" + testNum + " WHERE (deviceId = 'd0' or deviceId = 'd1')"); + int[] expectedPointNumOfDevice = new int[] {0, 0, 400, 400, 400}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain d4[1, 400] + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE (deviceId = 'd1' or deviceId = 'd2' or deviceId = 'd3')"); + expectedPointNumOfDevice = new int[] {0, 0, 0, 0, 400}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + // remain nothing + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE (deviceId = 'd2' or deviceId = 'd3' or deviceId = 'd4' or deviceId = 'd0')"); + expectedPointNumOfDevice = new int[] {0, 0, 0, 0, 0}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + + /// remain nothing + statement.execute( + "DELETE FROM vehicle" + + testNum + + " WHERE (deviceId = 'd2' or deviceId = 'd3' or deviceId = 'd4' or deviceId = 'd0' or deviceId = 'd1')"); + expectedPointNumOfDevice = new int[] {0, 0, 0, 0, 0}; + checkDevicePoint(expectedPointNumOfDevice, statement, testNum); + } + cleanData(testNum); + } + + @Category(ManualIT.class) + @Test + public void testRepeatedlyWriteAndDeletion() throws SQLException { + int testNum = 19; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + } + // repeat 100 times + // each time write 10000 points and delete 1000 of them randomly + int repetition = 100; + Random random = new Random(); + + for (int rep = 0; rep < repetition; rep++) { + int fileNumMax = 100; + int pointPerFile = 100; + int deletionRange = 1000; + long time = -1; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("create database if not exists test"); + statement.execute("use test"); + + statement.execute( + "create table if not exists table" + testNum + "(deviceId STRING TAG, s0 INT32 field)"); + + for (int i = 1; i <= fileNumMax; i++) { + for (int j = 0; j < pointPerFile; j++) { + statement.execute( + String.format( + "INSERT INTO test.table" + testNum + "(time, deviceId, s0) VALUES(%d,'d0',%d)", + time + 1, + time + 1)); + time++; + } + statement.execute("FLUSH"); + } + + int totalPointNum = fileNumMax * pointPerFile; + long deletionStart = random.nextInt((int) time); + long deletionEnd = Math.min(deletionStart + deletionRange, time); + long pointDeleted = deletionEnd - deletionStart + 1; + LOGGER.info("{}: deletion range [{}, {}]", rep, deletionStart, deletionEnd); + + statement.execute( + "delete from test.table" + + testNum + + " where time >= " + + deletionStart + + " and time <= " + + deletionEnd); + + // check the point count + try (ResultSet set = + statement.executeQuery( + "select count(*) from table" + testNum + " where time < " + totalPointNum)) { + assertTrue(set.next()); + long expectedCnt = totalPointNum - pointDeleted; + if (expectedCnt != set.getLong(1)) { + List remainingRanges = collectDataRanges(statement, time, testNum); + LOGGER.info("{}: Remaining ranges: {}", rep, remainingRanges); + fail( + String.format( + "Inconsistent number of points %d - %d", expectedCnt, set.getLong(1))); + } + } + } + } + } + + @Test + public void testMergeDeletion() throws SQLException { + int testNum = 20; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists db1"); + statement.execute("use db1"); + statement.execute( + "create table t" + + testNum + + "(country tag,region tag, city tag, device tag, ab1 ATTRIBUTE, s1 int32, s2 float, s3 boolean, s4 string)"); + statement.execute( + "INSERT INTO t" + + testNum + + "(time,country,region,city,device,ab1,s1,s2,s3,s4) values (100,'china','hebei','shijiazhuang','d1','ab1',1,1,1,1),(200,null,'hebei','shijiazhuang','d2','ab2',1,1,1,1),(300,'china','beijing','beijing','d1','ab3',1,1,1,1),(400,'china','tianjin','tianjin','d1','ab4',1,1,1,1),(500,'china','sichuan','chengdu','d1',null,1,1,1,1),(600,'china','zhejiang','hangzhou','d1','ab6',1,1,1,1),(700,'japan','dao','tokyo','d1','ab7',1,1,1,1),(800,'canada','tronto','shijiazhuang','d1','ab8',null,1,1,1),(900,'usa','ca','oldmountain','d1','ab9',1,1,1,1),(1000,'tailand',null,'mangu','d1','ab10',1,1,1,1),(1100,'china','hebei','','d1','ab11',1,1,1,1),(1200,'','hebei','','d1','ab12',1,1,1,1),(1300,'china','','','d1','ab13',1,1,1,1)"); + statement.execute("flush"); + int cnt = 0; + try (ResultSet set = + statement.executeQuery( + "select time,country,region,city,device,ab1,s1,s2,s3,s4 from t" + + testNum + + " order by time")) { + while (set.next()) { + cnt++; + } + assertEquals(13, cnt); + } + cnt = 0; + statement.execute("delete from t" + testNum + " where country='japan'"); + try (ResultSet set = + statement.executeQuery( + "select time,country,region,city,device,ab1,s1,s2,s3,s4 from t" + + testNum + + " order by time")) { + while (set.next()) { + cnt++; + } + assertEquals(12, cnt); + } + cnt = 0; + statement.execute("delete from t" + testNum + " where country='china' and region='beijing'"); + try (ResultSet set = + statement.executeQuery( + "select time,country,region,city,device,ab1,s1,s2,s3,s4 from t" + + testNum + + " order by time")) { + while (set.next()) { + cnt++; + } + assertEquals(11, cnt); + } + } + } + + @Category(ManualIT.class) + @Test + public void testConcurrentFlushAndSequentialDeletion() + throws InterruptedException, ExecutionException, SQLException { + int testNum = 21; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + statement.execute( + "SET CONFIGURATION inner_compaction_task_selection_mods_file_threshold='1024'"); + } + + AtomicLong writtenPointCounter = new AtomicLong(-1); + ExecutorService threadPool = Executors.newCachedThreadPool(); + int fileNumMax = 1000; + int pointPerFile = 1000; + int deviceNum = 4; + Future writeThread = + threadPool.submit( + () -> + write( + writtenPointCounter, + threadPool, + fileNumMax, + pointPerFile, + deviceNum, + testNum, + true)); + int deletionRange = 150; + int deletionInterval = 1500; + Future deletionThread = + threadPool.submit( + () -> + sequentialDeletion( + writtenPointCounter, + threadPool, + deletionRange, + deletionInterval, + fileNumMax * pointPerFile - 1, + testNum)); + writeThread.get(); + deletionThread.get(); + threadPool.shutdown(); + boolean success = threadPool.awaitTermination(1, TimeUnit.MINUTES); + assertTrue(success); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + } + } + + @Category(ManualIT.class) + @Test + public void testConcurrentFlushAndRandomDeletion() + throws InterruptedException, ExecutionException, SQLException { + int testNum = 22; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + statement.execute( + "SET CONFIGURATION inner_compaction_task_selection_mods_file_threshold='1024'"); + } + + AtomicLong writtenPointCounter = new AtomicLong(-1); + AtomicLong deletedPointCounter = new AtomicLong(0); + int fileNumMax = 1000; + int pointPerFile = 1000; + int deviceNum = 4; + ExecutorService threadPool = Executors.newCachedThreadPool(); + Future writeThread = + threadPool.submit( + () -> + write( + writtenPointCounter, + threadPool, + fileNumMax, + pointPerFile, + deviceNum, + testNum, + true)); + int deletionRange = 100; + int minIntervalToRecord = 1000; + Future deletionThread = + threadPool.submit( + () -> + randomDeletion( + writtenPointCounter, + deletedPointCounter, + threadPool, + fileNumMax, + pointPerFile, + deletionRange, + minIntervalToRecord, + testNum)); + writeThread.get(); + deletionThread.get(); + threadPool.shutdown(); + boolean success = threadPool.awaitTermination(1, TimeUnit.MINUTES); + assertTrue(success); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + } + } + + @Category(ManualIT.class) + @Test + public void testConcurrentFlushAndRandomDeletionWithRestart() + throws InterruptedException, ExecutionException, SQLException { + int testNum = 23; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + } + + AtomicLong writtenPointCounter = new AtomicLong(-1); + AtomicLong deletedPointCounter = new AtomicLong(0); + ExecutorService writeDeletionThreadPool = Executors.newCachedThreadPool(); + ExecutorService restartThreadPool = Executors.newCachedThreadPool(); + int fileNumMax = 1000; + int pointPerFile = 1000; + int deviceNum = 4; + Future writeThread = + writeDeletionThreadPool.submit( + () -> + write( + writtenPointCounter, + writeDeletionThreadPool, + fileNumMax, + pointPerFile, + deviceNum, + testNum, + true)); + int deletionRange = 100; + int minIntervalToRecord = 1000; + Future deletionThread = + writeDeletionThreadPool.submit( + () -> + randomDeletion( + writtenPointCounter, + deletedPointCounter, + writeDeletionThreadPool, + fileNumMax, + pointPerFile, + deletionRange, + minIntervalToRecord, + testNum)); + int restartTargetPointWritten = 100000; + Future restartThread = + restartThreadPool.submit( + () -> restart(writtenPointCounter, restartTargetPointWritten, writeDeletionThreadPool)); + try { + writeThread.get(); + } catch (CancellationException ignored) { + + } + try { + deletionThread.get(); + } catch (CancellationException ignored) { + + } + restartThread.get(); + writeDeletionThreadPool.shutdown(); + boolean success = writeDeletionThreadPool.awaitTermination(1, TimeUnit.MINUTES); + assertTrue(success); + + // test that should be written are written, deleted are deleted + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + try (ResultSet set = + statement.executeQuery( + "select count(*) from table" + + testNum + + " where time < " + + writtenPointCounter.get())) { + assertTrue(set.next()); + assertEquals(writtenPointCounter.get() - deletedPointCounter.get(), set.getLong(1)); + } + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + } + } + + private Void write( + AtomicLong writtenPointCounter, + ExecutorService allThreads, + int fileNumMax, + int pointPerFile, + int deviceNum, + int testNum, + boolean roundRobinDevice) + throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("create database if not exists test"); + statement.execute("use test"); + + statement.execute( + "create table if not exists table" + + testNum + + "(city TAG, deviceId STRING TAG, s0 INT32 field)"); + + for (int i = 1; i <= fileNumMax; i++) { + for (int j = 0; j < pointPerFile; j++) { + long time = writtenPointCounter.get() + 1; + if (roundRobinDevice) { + statement.execute( + String.format( + "INSERT INTO test.table" + + testNum + + "(time, city, deviceId, s0) VALUES(%d, 'bj', 'd" + + (time % deviceNum) + + "',%d)", + time, + time)); + } else { + for (int d = 0; d < deviceNum; d++) { + statement.execute( + String.format( + "INSERT INTO test.table" + + testNum + + "(time, city, deviceId, s0) VALUES(%d, 'bj', 'd" + + d + + "',%d)", + time, + time)); + } + } + + writtenPointCounter.incrementAndGet(); + if (Thread.interrupted()) { + return null; + } + } + statement.execute("FLUSH"); + if (i % 100 == 0) { + LOGGER.info("{} files written", i); + } + } + } catch (SQLException e) { + if (e.getMessage().contains("Fail to reconnect")) { + // restart triggered, ignore + return null; + } else { + allThreads.shutdownNow(); + throw e; + } + } catch (Throwable e) { + allThreads.shutdownNow(); + throw e; + } + return null; + } + + private Void sequentialDeletion( + AtomicLong writtenPointCounter, + ExecutorService allThreads, + int deletionRange, + int deletionInterval, + long deletionEnd, + int testNum) + throws SQLException, InterruptedException { + // delete every 10 points in 100 points + int deletionOffset = 0; + long nextPointNumToDelete = deletionInterval; + // pointPerFile * fileNumMax + + long deletedCnt = 0; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("create database if not exists test"); + statement.execute("use test"); + while (deletionOffset < deletionEnd + && nextPointNumToDelete < deletionEnd + && !Thread.interrupted()) { + if (writtenPointCounter.get() >= nextPointNumToDelete) { + statement.execute( + "delete from test.table" + + testNum + + " where time >= " + + deletionOffset + + " and time < " + + (deletionOffset + deletionRange)); + deletedCnt += deletionRange; + LOGGER.info("{} points deleted", deletedCnt); + + try (ResultSet set = + statement.executeQuery( + "select count(*) from table" + + testNum + + " where time < " + + nextPointNumToDelete)) { + assertTrue(set.next()); + assertEquals(nextPointNumToDelete * 9 / 10, set.getLong(1)); + } + deletionOffset += deletionInterval; + nextPointNumToDelete += deletionInterval; + + } else { + Thread.sleep(10); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (SQLException e) { + if (e.getMessage().contains("Fail to reconnect")) { + // restart triggered, ignore + return null; + } else { + allThreads.shutdownNow(); + throw e; + } + } catch (Throwable e) { + allThreads.shutdownNow(); + throw e; + } + return null; + } + + private Void randomDeletion( + AtomicLong writtenPointCounter, + AtomicLong deletedPointCounter, + ExecutorService allThreads, + int fileNumMax, + int pointPerFile, + int deletionRange, + int minIntervalToRecord, + int testNum) + throws SQLException, InterruptedException { + // delete random 100 points each time + List undeletedRanges = new ArrayList<>(); + // pointPerFile * fileNumMax + long deletionEnd = (long) fileNumMax * pointPerFile - 1; + long nextRangeStart = 0; + Random random = new Random(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("create database if not exists test"); + statement.execute("use test"); + while ((writtenPointCounter.get() < deletionEnd || !undeletedRanges.isEmpty()) + && !Thread.interrupted()) { + // record the newly inserted interval if it is long enough + long currentWrittenTime = writtenPointCounter.get(); + if (currentWrittenTime - nextRangeStart >= minIntervalToRecord) { + undeletedRanges.add(new TimeRange(nextRangeStart, currentWrittenTime)); + nextRangeStart = currentWrittenTime + 1; + } + if (undeletedRanges.isEmpty()) { + Thread.sleep(10); + continue; + } + // pick up a random range + int rangeIndex = random.nextInt(undeletedRanges.size()); + TimeRange timeRange = undeletedRanges.get(rangeIndex); + // delete a random part in the range + LOGGER.info("Pick up a range [{}, {}]", timeRange.getMin(), timeRange.getMax()); + long rangeDeletionStart; + long timeRangeLength = timeRange.getMax() - timeRange.getMin() + 1; + if (timeRangeLength == 1) { + rangeDeletionStart = timeRange.getMin(); + } else { + rangeDeletionStart = random.nextInt((int) (timeRangeLength - 1)) + timeRange.getMin(); + } + long rangeDeletionEnd = Math.min(rangeDeletionStart + deletionRange, timeRange.getMax()); + LOGGER.info("Deletion range [{}, {}]", rangeDeletionStart, rangeDeletionEnd); + + statement.execute( + "delete from test.table" + + testNum + + " where time >= " + + rangeDeletionStart + + " and time <= " + + rangeDeletionEnd); + deletedPointCounter.addAndGet(rangeDeletionEnd - rangeDeletionStart + 1); + LOGGER.info( + "Deleted range [{}, {}], written points: {}, deleted points: {}", + timeRange.getMin(), + timeRange.getMax(), + currentWrittenTime + 1, + deletedPointCounter.get()); + + // update the range + if (rangeDeletionStart == timeRange.getMin() && rangeDeletionEnd == timeRange.getMax()) { + // range fully deleted + undeletedRanges.remove(rangeIndex); + } else if (rangeDeletionStart == timeRange.getMin()) { + // prefix deleted + timeRange.setMin(rangeDeletionEnd + 1); + } else if (rangeDeletionEnd == timeRange.getMax()) { + // suffix deleted + timeRange.setMax(rangeDeletionStart - 1); + } else { + // split into two ranges + undeletedRanges.add(new TimeRange(rangeDeletionEnd + 1, timeRange.getMax())); + timeRange.setMax(rangeDeletionStart - 1); + } + + // check the point count + try (ResultSet set = + statement.executeQuery( + "select count(*) from table" + testNum + " where time <= " + currentWrittenTime)) { + assertTrue(set.next()); + long expectedCnt = currentWrittenTime + 1 - deletedPointCounter.get(); + if (expectedCnt != set.getLong(1)) { + undeletedRanges = mergeRanges(undeletedRanges); + List remainingRanges = + collectDataRanges(statement, currentWrittenTime, testNum); + LOGGER.info("Expected ranges: {}", undeletedRanges); + LOGGER.info("Remaining ranges: {}", remainingRanges); + fail( + String.format( + "Inconsistent number of points %d - %d", expectedCnt, set.getLong(1))); + } + } + + Thread.sleep(10); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (SQLException e) { + if (e.getMessage().contains("Fail to reconnect")) { + // restart triggered, ignore + return null; + } else { + allThreads.shutdownNow(); + throw e; + } + } catch (ParallelRequestTimeoutException ignored) { + // restart triggered, ignore + return null; + } catch (Throwable e) { + allThreads.shutdownNow(); + throw e; + } + return null; + } + + private Void randomDeviceDeletion( + AtomicLong writtenPointCounter, + List deviceDeletedPointCounters, + ExecutorService allThreads, + int fileNumMax, + int pointPerFile, + int deletionRange, + int minIntervalToRecord, + int testNum) + throws SQLException, InterruptedException { + // delete random 'deletionRange' points each time + List> allDeviceUndeletedRanges = new ArrayList<>(); + for (int i = 0; i < deviceDeletedPointCounters.size(); i++) { + allDeviceUndeletedRanges.add(new ArrayList<>()); + } + // pointPerFile * fileNumMax + long deletionEnd = (long) fileNumMax * pointPerFile - 1; + long nextRangeStart = 0; + Random random = new Random(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("create database if not exists test"); + statement.execute("use test"); + while ((writtenPointCounter.get() < deletionEnd + || allDeviceUndeletedRanges.stream().anyMatch(l -> !l.isEmpty())) + && !Thread.interrupted()) { + // record the newly inserted interval if it is long enough + for (int i = 0; i < deviceDeletedPointCounters.size(); i++) { + long currentWrittenTime = writtenPointCounter.get(); + List deviceUndeletedRanges = allDeviceUndeletedRanges.get(i); + + if (currentWrittenTime - nextRangeStart >= minIntervalToRecord) { + deviceUndeletedRanges.add(new TimeRange(nextRangeStart, currentWrittenTime)); + nextRangeStart = currentWrittenTime + 1; + } + if (deviceUndeletedRanges.isEmpty()) { + Thread.sleep(10); + continue; + } + // pick up a random range + int rangeIndex = random.nextInt(deviceUndeletedRanges.size()); + TimeRange timeRange = deviceUndeletedRanges.get(rangeIndex); + // delete a random part in the range + LOGGER.debug("Pick up a range [{}, {}]", timeRange.getMin(), timeRange.getMax()); + long rangeDeletionStart; + long timeRangeLength = timeRange.getMax() - timeRange.getMin() + 1; + if (timeRangeLength == 1) { + rangeDeletionStart = timeRange.getMin(); + } else { + rangeDeletionStart = random.nextInt((int) (timeRangeLength - 1)) + timeRange.getMin(); + } + long rangeDeletionEnd = Math.min(rangeDeletionStart + deletionRange, timeRange.getMax()); + LOGGER.debug("Deletion range [{}, {}]", rangeDeletionStart, rangeDeletionEnd); + + statement.execute( + "delete from test.table" + + testNum + + " where time >= " + + rangeDeletionStart + + " and time <= " + + rangeDeletionEnd + + " and deviceId = 'd" + + i + + "'"); + deviceDeletedPointCounters.get(i).addAndGet(rangeDeletionEnd - rangeDeletionStart + 1); + LOGGER.debug( + "Deleted range [{}, {}], written points: {}, deleted points: {}", + timeRange.getMin(), + timeRange.getMax(), + currentWrittenTime + 1, + deviceDeletedPointCounters.get(i).get()); + + // update the range + if (rangeDeletionStart == timeRange.getMin() && rangeDeletionEnd == timeRange.getMax()) { + // range fully deleted + deviceUndeletedRanges.remove(rangeIndex); + } else if (rangeDeletionStart == timeRange.getMin()) { + // prefix deleted + timeRange.setMin(rangeDeletionEnd + 1); + } else if (rangeDeletionEnd == timeRange.getMax()) { + // suffix deleted + timeRange.setMax(rangeDeletionStart - 1); + } else { + // split into two ranges + deviceUndeletedRanges.add(new TimeRange(rangeDeletionEnd + 1, timeRange.getMax())); + timeRange.setMax(rangeDeletionStart - 1); + } + + // check the point count + try (ResultSet set = + statement.executeQuery( + "select count(*) from table" + + testNum + + " where time <= " + + currentWrittenTime + + " AND deviceId = 'd" + + i + + "'")) { + assertTrue(set.next()); + long expectedCnt = currentWrittenTime + 1 - deviceDeletedPointCounters.get(i).get(); + if (expectedCnt != set.getLong(1)) { + allDeviceUndeletedRanges.set(i, mergeRanges(deviceUndeletedRanges)); + List remainingRanges = + collectDataRanges(statement, currentWrittenTime, testNum); + LOGGER.debug("Expected ranges: {}", deviceUndeletedRanges); + LOGGER.debug("Remaining ranges: {}", remainingRanges); + fail( + String.format( + "Inconsistent number of points %d - %d", expectedCnt, set.getLong(1))); + } + } + + Thread.sleep(10); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (SQLException e) { + if (e.getMessage().contains("Fail to reconnect")) { + // restart triggered, ignore + return null; + } else { + allThreads.shutdownNow(); + throw e; + } + } catch (ParallelRequestTimeoutException ignored) { + // restart triggered, ignore + return null; + } catch (Throwable e) { + allThreads.shutdownNow(); + throw e; + } + return null; + } + + private Void restart( + AtomicLong writtenPointCounter, long targetPointNum, ExecutorService threadPool) + throws InterruptedException, SQLException { + while (writtenPointCounter.get() < targetPointNum) { + Thread.sleep(10); + } + threadPool.shutdownNow(); + threadPool.awaitTermination(1, TimeUnit.MINUTES); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("flush"); + } + + TestUtils.restartDataNodes(); + return null; + } + + private List mergeRanges(List timeRanges) { + timeRanges.sort(null); + List result = new ArrayList<>(); + TimeRange current = null; + for (TimeRange timeRange : timeRanges) { + if (current == null) { + current = timeRange; + } else { + if (current.getMax() == timeRange.getMin() - 1) { + current.setMax(timeRange.getMax()); + } else { + result.add(current); + current = timeRange; + } + } + } + result.add(current); + return result; + } + + private List collectDataRanges(Statement statement, long timeUpperBound, int testNum) + throws SQLException { + List ranges = new ArrayList<>(); + try (ResultSet set = + statement.executeQuery( + "select time from table" + testNum + " where time <= " + timeUpperBound)) { + while (set.next()) { + long time = set.getLong(1); + if (ranges.isEmpty()) { + ranges.add(new TimeRange(time, time)); + } else { + TimeRange lastRange = ranges.get(ranges.size() - 1); + if (lastRange.getMax() == time - 1) { + lastRange.setMax(time); + } else { + ranges.add(new TimeRange(time, time)); + } + } + } + } + return ranges; + } + + @Test + public void deleteTableOfTheSameNameTest() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 24; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db2"); + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db3"); + + session.executeNonQueryStatement( + "CREATE TABLE db1.table" + testNum + " (id1 string tag, m1 int32 field)"); + session.executeNonQueryStatement( + "INSERT INTO db1.table" + testNum + " (time, id1, m1) VALUES (1, 'd1', 1)"); + + session.executeNonQueryStatement( + "CREATE TABLE db2.table" + testNum + " (id1 string tag, m1 int32 field)"); + session.executeNonQueryStatement( + "INSERT INTO db2.table" + testNum + " (time, id1, m1) VALUES (2, 'd2', 2)"); + + session.executeNonQueryStatement( + "CREATE TABLE db3.table" + testNum + " (id1 string tag, m1 int32 field)"); + session.executeNonQueryStatement( + "INSERT INTO db3.table" + testNum + " (time, id1, m1) VALUES (3, 'd3', 3)"); + + session.executeNonQueryStatement("USE db2"); + session.executeNonQueryStatement("DELETE FROM table" + testNum); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertEquals(1, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + dataSet = + session.executeQueryStatement("select * from db2.table" + testNum + " order by time"); + assertFalse(dataSet.hasNext()); + + dataSet = + session.executeQueryStatement("select * from db3.table" + testNum + " order by time"); + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertEquals("d3", rec.getFields().get(1).toString()); + assertEquals(3, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DELETE FROM db3.table" + testNum); + + dataSet = + session.executeQueryStatement("select * from db1.table" + testNum + " order by time"); + rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertEquals(1, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + dataSet = + session.executeQueryStatement("select * from db2.table" + testNum + " order by time"); + assertFalse(dataSet.hasNext()); + + dataSet = + session.executeQueryStatement("select * from db3.table" + testNum + " order by time"); + assertFalse(dataSet.hasNext()); + } + } + + @Test + public void testConcurrentFlushAndRandomDeviceDeletion() + throws InterruptedException, ExecutionException, SQLException { + int testNum = 25; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + statement.execute( + "SET CONFIGURATION inner_compaction_task_selection_mods_file_threshold='1024'"); + statement.execute("SET CONFIGURATION inner_seq_performer='FAST'"); + } catch (Exception ignored) { + // remote mode cannot find the config file during SET CONFIGURATION + } + + AtomicLong writtenPointCounter = new AtomicLong(-1); + int fileNumMax = 100; + int pointPerFile = 100; + int deviceNum = 4; + List deviceDeletedPointCounters = new ArrayList<>(deviceNum); + for (int i = 0; i < deviceNum; i++) { + deviceDeletedPointCounters.add(new AtomicLong(0)); + } + + ExecutorService threadPool = Executors.newCachedThreadPool(); + Future writeThread = + threadPool.submit( + () -> + write( + writtenPointCounter, + threadPool, + fileNumMax, + pointPerFile, + deviceNum, + testNum, + false)); + int deletionRange = 100; + int minIntervalToRecord = 1000; + Future deletionThread = + threadPool.submit( + () -> + randomDeviceDeletion( + writtenPointCounter, + deviceDeletedPointCounters, + threadPool, + fileNumMax, + pointPerFile, + deletionRange, + minIntervalToRecord, + testNum)); + writeThread.get(); + deletionThread.get(); + threadPool.shutdown(); + boolean success = threadPool.awaitTermination(1, TimeUnit.MINUTES); + assertTrue(success); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists test"); + statement.execute("SET CONFIGURATION inner_seq_performer='read_chunk'"); + } + } + + @Test + public void testCaseSensitivity() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + session.executeNonQueryStatement("USE db1"); + session.executeNonQueryStatement("CREATE TABLE case_sensitivity (tag1 TAG, s1 INT32)"); + + session.executeNonQueryStatement( + "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (1, 'd1', 1)"); + session.executeNonQueryStatement( + "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (2, 'd2', 2)"); + session.executeNonQueryStatement( + "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (3, 'd3', 3)"); + + session.executeNonQueryStatement("DELETE FROM DB1.case_sensitivity where time = 1"); + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.case_sensitivity order by time"); + RowRecord rec = dataSet.next(); + assertEquals(2, rec.getFields().get(0).getLongV()); + assertEquals("d2", rec.getFields().get(1).toString()); + assertEquals(2, rec.getFields().get(2).getIntV()); + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertEquals("d3", rec.getFields().get(1).toString()); + assertEquals(3, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DELETE FROM db1.CASE_sensitivity where time = 2"); + dataSet = session.executeQueryStatement("select * from db1.case_sensitivity order by time"); + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertEquals("d3", rec.getFields().get(1).toString()); + assertEquals(3, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DELETE FROM db1.CASE_sensitivity where TAG1 = 'd3'"); + dataSet = session.executeQueryStatement("select * from db1.case_sensitivity order by time"); + assertFalse(dataSet.hasNext()); + } + } + + @Ignore("performance") + @Test + public void testDeletionWritePerformance() throws SQLException, IOException { + int fileNumMax = 10000; + int fileNumStep = 100; + int deletionRepetitions = 10; + List fileNumsRecorded = new ArrayList<>(); + List timeConsumptionNsRecorded = new ArrayList<>(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION enable_seq_space_compaction='false'"); + + statement.execute("create database if not exists test"); + statement.execute("use test"); + + statement.execute("create table table1(deviceId STRING TAG, s0 INT32 FIELD)"); + + for (int i = 1; i <= fileNumMax; i++) { + statement.execute( + String.format("INSERT INTO test.table1(time, deviceId, s0) VALUES(%d,'d0',%d)", i, i)); + statement.execute("FLUSH"); + + if (i % fileNumStep == 0) { + long start = System.nanoTime(); + for (int j = 0; j < deletionRepetitions; j++) { + statement.execute("DELETE FROM test.table1 WHERE deviceId = 'd0'"); + } + long end = System.nanoTime(); + fileNumsRecorded.add(i); + long timeConsumption = (end - start) / deletionRepetitions; + timeConsumptionNsRecorded.add(timeConsumption); + + System.out.println(i + "," + timeConsumption); + } + } + } + + try (BufferedWriter writer = new BufferedWriter(new FileWriter("test.txt"))) { + writer.write( + fileNumsRecorded.stream().map(i -> Integer.toString(i)).collect(Collectors.joining(","))); + writer.write("\n"); + writer.write( + timeConsumptionNsRecorded.stream() + .map(i -> Long.toString(i)) + .collect(Collectors.joining(","))); + writer.flush(); + } + } + + @Ignore("performance") + @Test + public void testDeletionReadPerformance() throws SQLException, IOException { + int fileNumMax = 100; + int pointNumPerFile = 100; + int deletionNumStep = 100; + int maxDeletionNum = 10000; + int readRepetitions = 5; + List deletionNumsRecorded = new ArrayList<>(); + List timeConsumptionNsRecorded = new ArrayList<>(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION enable_seq_space_compaction='false'"); + + statement.execute("create database if not exists test"); + statement.execute("use test"); + + statement.execute("create table table1(deviceId STRING TAG, s0 INT32 FIELD)"); + + for (int i = 1; i <= fileNumMax; i++) { + for (int j = 0; j < pointNumPerFile; j++) { + long timestamp = (long) (i - 1) * pointNumPerFile + j; + statement.execute( + String.format( + "INSERT INTO test.table1(time, deviceId, s0) VALUES(%d,'d0',%d)", + timestamp, timestamp)); + } + statement.execute("FLUSH"); + } + + for (int i = 1; i <= maxDeletionNum; i++) { + statement.execute("DELETE FROM test.table1 WHERE deviceId = 'd0'"); + if (i % deletionNumStep == 0) { + long start = System.nanoTime(); + for (int j = 0; j < readRepetitions; j++) { + ResultSet resultSet = statement.executeQuery("SELECT * FROM test.table1"); + //noinspection StatementWithEmptyBody + while (resultSet.next()) { + // just iterate the set + } + } + long end = System.nanoTime(); + long timeConsumption = (end - start) / readRepetitions; + timeConsumptionNsRecorded.add(timeConsumption); + deletionNumsRecorded.add(i); + System.out.println(i + "," + timeConsumption); + } + } + } + + try (BufferedWriter writer = new BufferedWriter(new FileWriter("test.txt"))) { + writer.write( + deletionNumsRecorded.stream() + .map(i -> Integer.toString(i)) + .collect(Collectors.joining(","))); + writer.write("\n"); + writer.write( + timeConsumptionNsRecorded.stream() + .map(i -> Long.toString(i)) + .collect(Collectors.joining(","))); + writer.flush(); + } + } + + private static void prepareDatabase() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.execute(sql); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + + private void prepareData(int testNum, int deviceNum) throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + String.format( + "CREATE TABLE IF NOT EXISTS vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + for (int d = 0; d < deviceNum; d++) { + // prepare seq file + for (int i = 201; i <= 300; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, d, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + } + + statement.execute("flush"); + + for (int d = 0; d < deviceNum; d++) { + // prepare unseq File + for (int i = 1; i <= 100; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, d, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + } + statement.execute("flush"); + + for (int d = 0; d < deviceNum; d++) { + // prepare BufferWrite cache + for (int i = 301; i <= 400; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, d, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + // prepare Overflow cache + for (int i = 101; i <= 200; i++) { + statement.execute( + String.format( + insertTemplate, testNum, i, d, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + } + } + } + } + + private void cleanData(int testNum) throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + String deleteAllTemplate = "DROP TABLE IF EXISTS vehicle%d"; + statement.execute(String.format(deleteAllTemplate, testNum)); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBExecuteBatchTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBExecuteBatchTableIT.java new file mode 100644 index 0000000000000..a6d71ff526867 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBExecuteBatchTableIT.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +@Ignore // 'Drop Table' and 'Alter table' is not supported +public class IoTDBExecuteBatchTableIT { + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testJDBCExecuteBatch() { + + try (Connection connection = EnvFactory.getEnv().getConnection(TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.addBatch("create database ln"); + statement.addBatch("USE \"ln\""); + statement.addBatch("create table wf01 (tag1 string tag, temprature double field)"); + statement.addBatch( + "insert into wf01(tag1,time,temperature) values(\'wt01\', 1509465600000,1.2)"); + statement.addBatch( + "insert into wf01(tag1,time,temperature) values(\'wt01\', 1509465600001,2.3)"); + + statement.addBatch("drop table wf01"); + statement.addBatch("create table wf01 (tag1 string tag, temprature double field)"); + + statement.addBatch( + "insert into wf01(tag1,time,temperature) values(\'wt01\', 1509465600002,3.4)"); + statement.executeBatch(); + statement.clearBatch(); + ResultSet resultSet = statement.executeQuery("select * from wf01"); + int count = 0; + + String[] timestamps = {"1509465600002"}; + String[] values = {"3.4"}; + + while (resultSet.next()) { + assertEquals(timestamps[count], resultSet.getString("time")); + assertEquals(values[count], resultSet.getString("temperature")); + count++; + } + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + @Test + public void testJDBCExecuteBatchForCreateMultiTimeSeriesPlan() { + try (Connection connection = EnvFactory.getEnv().getConnection(TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(100); + statement.execute("create database ln"); + statement.execute("USE \"ln\""); + statement.addBatch("create table wf01 (tag1 string tag, temprature double field)"); + + statement.addBatch( + "insert into wf01(tag1,time,temperature) values(\'wt01\', 1509465600000,1.2)"); + statement.addBatch( + "insert into wf01(tag1,time,temperature) values(\'wt01\', 1509465600001,2.3)"); + statement.addBatch("drop table wf01"); + + statement.addBatch( + "create table turbine (tag1 string tag, attr1 string attribute, attr2 string attribute, s1 boolean field, s2 float field)"); + + statement.addBatch("create table wf01 (tag1 string tag, temprature double field)"); + statement.addBatch( + "insert into wf01(tag1,time,temperature) values(\'wt01\', 1509465600002,3.4)"); + statement.addBatch("alter table turbine add column s3 boolean field"); + statement.executeBatch(); + statement.clearBatch(); + ResultSet resultSet = statement.executeQuery("select * from wf01"); + String[] timestamps = {"1509465600002"}; + String[] values = {"3.4"}; + int count = 0; + while (resultSet.next()) { + assertEquals(timestamps[count], resultSet.getString("time")); + assertEquals(values[count], resultSet.getString("temperature")); + count++; + } + ResultSet timeSeriesResultSetForS1 = statement.executeQuery("describe turbine"); + count = 0; + String[] keys = {"ColumnName", "DataType", "Category"}; + String[][] value_columns = { + new String[] {"Time", "TIMESTAMP", "TIME"}, + new String[] {"tag1", "STRING", "TAG"}, + new String[] {"attr1", "STRING", "ATTRIBUTE"}, + new String[] {"attr2", "STRING", "ATTRIBUTE"}, + new String[] {"s1", "BOOLEAN", "MEASUREMENT"}, + new String[] {"s2", "FLOAT", "MEASUREMENT"}, + }; + + while (timeSeriesResultSetForS1.next()) { + for (int i = 0; i < keys.length; i++) { + assertEquals(value_columns[count][i], timeSeriesResultSetForS1.getString(keys[i])); + } + count++; + } + } catch (SQLException e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java new file mode 100644 index 0000000000000..85af3ad0ff7c0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBFlushQueryTableIT.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFlushQueryTableIT { + + private static String[] sqls = + new String[] { + "CREATE DATABASE test", + "USE \"test\"", + "CREATE TABLE vehicle (tag1 string tag, s0 int32 field)", + "insert into vehicle(tag1,time,s0) values('d0',1,101)", + "insert into vehicle(tag1,time,s0) values('d0',2,198)", + "insert into vehicle(tag1,time,s0) values('d0',100,99)", + "insert into vehicle(tag1,time,s0) values('d0',101,99)", + "insert into vehicle(tag1,time,s0) values('d0',102,80)", + "insert into vehicle(tag1,time,s0) values('d0',103,99)", + "insert into vehicle(tag1,time,s0) values('d0',104,90)", + "insert into vehicle(tag1,time,s0) values('d0',105,99)", + "insert into vehicle(tag1,time,s0) values('d0',106,99)", + "flush", + "insert into vehicle(tag1,time,s0) values('d0',2,10000)", + "insert into vehicle(tag1,time,s0) values('d0',50,10000)", + "insert into vehicle(tag1,time,s0) values('d0',1000,22222)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + System.out.println(sql); + statement.execute(sql); + } + } catch (Exception e) { + fail("insertData failed."); + } + } + + @Test + public void selectAllSQLTest() { + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE \"test\""); + try (ResultSet resultSet = statement.executeQuery("SELECT * FROM vehicle"); ) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testFlushGivenGroup() { + String insertTemplate = + "INSERT INTO vehicle(tag1, time, s1, s2, s3) VALUES (%s, %d, %d, %f, %s)"; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("CREATE DATABASE group1"); + statement.execute("CREATE DATABASE group2"); + statement.execute("CREATE DATABASE group3"); + + for (int i = 1; i <= 3; i++) { + statement.execute(String.format("USE \"group%d\"", i)); + statement.execute( + "CREATE TABLE vehicle (tag1 string tag, s1 int32 field, s2 float field, s3 string field)"); + for (int j = 10; j < 20; j++) { + statement.execute(String.format(Locale.CHINA, insertTemplate, i, j, j, j * 0.1, j)); + } + } + statement.execute("FLUSH"); + + for (int i = 1; i <= 3; i++) { + statement.execute(String.format("USE \"group%d\"", i)); + for (int j = 0; j < 10; j++) { + statement.execute(String.format(Locale.CHINA, insertTemplate, i, j, j, j * 0.1, j)); + } + } + statement.execute("FLUSH group1"); + statement.execute("FLUSH group2,group3"); + + for (int i = 1; i <= 3; i++) { + statement.execute(String.format("USE \"group%d\"", i)); + for (int j = 0; j < 30; j++) { + statement.execute(String.format(Locale.CHINA, insertTemplate, i, j, j, j * 0.1, j)); + } + } + statement.execute("FLUSH group1 TRUE"); + statement.execute("FLUSH group2,group3 FALSE"); + + for (int i = 1; i <= 3; i++) { + statement.execute(String.format("USE \"group%d\"", i)); + int count = 0; + try (ResultSet resultSet = statement.executeQuery("SELECT * FROM vehicle")) { + while (resultSet.next()) { + count++; + } + } + assertEquals(30, count); + } + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testFlushGivenGroupNoData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE nodatagroup1"); + statement.execute("CREATE DATABASE nodatagroup2"); + statement.execute("CREATE DATABASE nodatagroup3"); + statement.execute("FLUSH nodatagroup1"); + statement.execute("FLUSH nodatagroup2"); + statement.execute("FLUSH nodatagroup3"); + statement.execute("FLUSH nodatagroup1, nodatagroup2"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testFlushNotExistGroupNoData() { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE noexist_nodatagroup1"); + try { + statement.execute("FLUSH noexist_nodatagroup1,notExistGroup1,notExistGroup2"); + } catch (final SQLException sqe) { + String expectedMsg = "Database notExistGroup1,notExistGroup2 does not exist"; + assertTrue(sqe.getMessage().contains(expectedMsg)); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testFlushTableAfterDropColumn() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database db1"); + statement.execute("use db1"); + statement.execute("create table t2(s1 text field, s2 text field)"); + statement.execute("insert into t2(time,s2) values(2,'t1')"); + statement.execute("alter table t2 drop column s2"); + statement.execute("flush"); + statement.execute("insert into t2(time,s1) values(2,'t1')"); + } catch (Exception e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertAlignedValuesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertAlignedValuesTableIT.java new file mode 100644 index 0000000000000..5740e8d42577c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertAlignedValuesTableIT.java @@ -0,0 +1,584 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBInsertAlignedValuesTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database t1"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInsertAlignedValues() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.addBatch( + "create table wf01 (tag1 string tag, status boolean field, temperature float field)"); + statement.addBatch( + "insert into wf01(tag1, time, status, temperature) values ('wt01', 4000, true, 17.1)"); + statement.addBatch( + "insert into wf01(tag1, time, status, temperature) values ('wt01', 5000, true, 20.1)"); + statement.addBatch( + "insert into wf01(tag1, time, status, temperature) values ('wt01', 6000, true, 22)"); + statement.executeBatch(); + + try (ResultSet resultSet = statement.executeQuery("select time, status from wf01")) { + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertFalse(resultSet.next()); + } + + try (ResultSet resultSet = + statement.executeQuery("select time, status, temperature from wf01")) { + + assertTrue(resultSet.next()); + assertEquals(4000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(17.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(5000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(20.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(6000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(22, resultSet.getDouble(3), 0.1); + + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testInsertAlignedNullableValues() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("use \"t1\""); + statement.addBatch( + "create table wf02 (tag1 string tag, status boolean field, temperature float field)"); + statement.addBatch( + "insert into wf02(tag1, time, status, temperature) values ('wt01', 4000, true, 17.1)"); + statement.addBatch("insert into wf02(tag1, time, status) values ('wt01', 5000, true)"); + statement.addBatch("insert into wf02(tag1, time, temperature) values ('wt01', 6000, 22)"); + statement.executeBatch(); + + try (ResultSet resultSet = statement.executeQuery("select status from wf02")) { + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(1)); + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(1)); + assertTrue(resultSet.next()); + resultSet.getBoolean(1); + assertTrue(resultSet.wasNull()); + assertFalse(resultSet.next()); + } + + try (ResultSet resultSet = + statement.executeQuery("select time, status, temperature from wf02")) { + + assertTrue(resultSet.next()); + assertEquals(4000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(17.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(5000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertNull(resultSet.getObject(3)); + + assertTrue(resultSet.next()); + assertEquals(6000, resultSet.getLong(1)); + assertNull(resultSet.getObject(2)); + assertEquals(22.0f, resultSet.getObject(3)); + + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testUpdatingAlignedValues() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.addBatch( + "create table wf03 (tag1 string tag, status boolean field, temperature float field)"); + statement.addBatch( + "insert into wf03(tag1, time, status, temperature) values ('wt01', 4000, true, 17.1)"); + statement.addBatch("insert into wf03(tag1, time, status) values ('wt01', 5000, true)"); + statement.addBatch("insert into wf03(tag1, time, temperature)values ('wt01', 5000, 20.1)"); + statement.addBatch("insert into wf03(tag1, time, temperature)values ('wt01', 6000, 22)"); + statement.executeBatch(); + + try (ResultSet resultSet = statement.executeQuery("select time, status from wf03")) { + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertTrue(resultSet.next()); + resultSet.getBoolean(2); + assertTrue(resultSet.wasNull()); + assertFalse(resultSet.next()); + } + + try (ResultSet resultSet = + statement.executeQuery("select time, status, temperature from wf03")) { + + assertTrue(resultSet.next()); + assertEquals(4000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(17.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(5000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(20.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(6000, resultSet.getLong(1)); + assertNull(resultSet.getObject(2)); + assertEquals(22.0f, resultSet.getObject(3)); + + assertFalse(resultSet.next()); + } + + statement.execute("flush"); + try (ResultSet resultSet = statement.executeQuery("select time, status from wf03")) { + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertTrue(resultSet.next()); + assertTrue(resultSet.getBoolean(2)); + assertTrue(resultSet.next()); + resultSet.getBoolean(2); + assertTrue(resultSet.wasNull()); + assertFalse(resultSet.next()); + } + + try (ResultSet resultSet = + statement.executeQuery("select time, status, temperature from wf03")) { + + assertTrue(resultSet.next()); + assertEquals(4000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(17.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(5000, resultSet.getLong(1)); + assertTrue(resultSet.getBoolean(2)); + assertEquals(20.1, resultSet.getDouble(3), 0.1); + + assertTrue(resultSet.next()); + assertEquals(6000, resultSet.getLong(1)); + assertNull(resultSet.getObject(2)); + assertEquals(22.0f, resultSet.getObject(3)); + + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testInsertAlignedValuesWithSameTimestamp() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.addBatch("create table sg3 (tag1 string tag, s2 double field, s1 double field)"); + statement.addBatch("insert into sg3(tag1,time,s2) values('d1',1,2)"); + statement.addBatch("insert into sg3(tag1,time,s1) values('d1',1,2)"); + statement.executeBatch(); + + try (ResultSet resultSet = statement.executeQuery("select time, s1, s2 from sg3")) { + + assertTrue(resultSet.next()); + assertEquals(1, resultSet.getLong(1)); + assertEquals(2.0d, resultSet.getObject(2)); + assertEquals(2.0d, resultSet.getObject(3)); + + assertFalse(resultSet.next()); + } + + statement.execute("flush"); + try (ResultSet resultSet = statement.executeQuery("select time, s1, s2 from sg3")) { + + assertTrue(resultSet.next()); + assertEquals(1, resultSet.getLong(1)); + assertEquals(2.0d, resultSet.getObject(2)); + assertEquals(2.0d, resultSet.getObject(3)); + + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testInsertWithWrongMeasurementNum1() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute( + "create table wf04 (tag1 string tag, status int32, temperature int32 field)"); + statement.execute( + "insert into wf04(tag1, time, status, temperature) values('wt01', 11000, 100)"); + fail(); + } catch (SQLException e) { + assertEquals( + "701: Inconsistent numbers of non-time column names and values: 3-2", e.getMessage()); + } + } + + @Test + public void testInsertWithWrongMeasurementNum2() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute( + "create table wf05 (tag1 string tag, status int32, temperature int32 field)"); + statement.execute( + "insert into wf05(tag1, time, status, temperature) values('wt01', 11000, 100, 300, 400)"); + fail(); + } catch (SQLException e) { + assertEquals( + "701: Inconsistent numbers of non-time column names and values: 3-4", e.getMessage()); + } + } + + @Test(expected = Exception.class) + public void testInsertWithWrongType() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute( + "create table dev6 (tag1 string tag, latitude int32 field, longitude int32 field)"); + statement.execute("insert into dev6(tag1,time,latitude,longitude) values('GPS', 1,1.3,6.7)"); + fail(); + } + } + + @Test + public void testInsertWithDuplicatedMeasurements() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute("create table wf07(tag1 string tag, s3 boolean field, status int32)"); + statement.execute( + "insert into wf07(tag1, time, s3, status, status) values('wt01', 100, true, 20.1, 20.2)"); + fail(); + } catch (SQLException e) { + assertTrue( + e.getMessage(), + e.getMessage().contains("Insertion contains duplicated measurement: status")); + } + } + + @Test + public void testInsertMultiRows() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute("create table sg8 (tag1 string tag, s1 int32 field, s2 int32 field)"); + statement.execute( + "insert into sg8(tag1, time, s1, s2) values('d1', 10, 2, 2), ('d1', 11, 3, '3'), ('d1', 12,12.11,false)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage(), e.getMessage().contains("data type is not consistent")); + } + } + + @Test + public void testInsertLargeNumber() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute("create table sg9 (tag1 string tag, s98 int64 field, s99 int64 field)"); + statement.execute( + "insert into sg9(tag1, time, s98, s99) values('d1', 10, 2, 271840880000000000000000)"); + fail("Exception expected"); + } catch (SQLException e) { + assertEquals( + "700: line 1:59: Invalid numeric literal: 271840880000000000000000", e.getMessage()); + } + } + + @Test + public void testInsertAlignedWithEmptyPage() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute( + "create table dev10 (tag1 string tag, s1 int32 field, s2 int32 field, s3 int32 field)"); + for (int i = 0; i < 100; i++) { + if (i == 99) { + statement.addBatch( + "insert into dev10(tag1,time,s1,s3) values(" + + "'GPS'" + + "," + + i + + "," + + i + + "," + + i + + ")"); + } else { + statement.addBatch( + "insert into dev10(tag1, time,s1,s2) values(" + + "'GPS'" + + "," + + i + + "," + + i + + "," + + i + + ")"); + } + } + statement.executeBatch(); + + statement.execute("flush"); + int rowCount = 0; + try (ResultSet resultSet = statement.executeQuery("select time, s3 from dev10")) { + while (resultSet.next()) { + int v = resultSet.getInt(2); + if (rowCount == 99) { + assertEquals(99, v); + } else { + assertTrue(resultSet.wasNull()); + } + rowCount++; + } + assertEquals(100, rowCount); + } + + try (ResultSet resultSet = statement.executeQuery("select time, s2 from dev10")) { + rowCount = 0; + while (resultSet.next()) { + int v = resultSet.getInt(2); + if (rowCount == 99) { + assertTrue(resultSet.wasNull()); + } else { + assertEquals(rowCount, v); + } + rowCount++; + } + assertEquals(100, rowCount); + } + + try (ResultSet resultSet = statement.executeQuery("select time, s1 from dev10")) { + rowCount = 0; + while (resultSet.next()) { + assertEquals(rowCount, resultSet.getInt(2)); + rowCount++; + } + assertEquals(100, rowCount); + } + } + } + + @Test + public void testInsertAlignedWithEmptyPage2() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute("create table sg11 (tag1 string tag, s1 string field, s2 string field)"); + + statement.execute("insert into sg11(tag1, time, s1, s2) values('d1', 1,'aa','bb')"); + statement.execute("insert into sg11(tag1, time, s1, s2) values('d1', 1,'aa','bb')"); + statement.execute("insert into sg11(tag1, time, s1, s2) values('d2', 1,'aa','bb')"); + statement.execute("flush"); + statement.execute("insert into sg11(tag1, time, s1, s2) values('d1', 1,'aa','bb')"); + } + } + + @Ignore // aggregation + @Test + public void testInsertComplexAlignedValues() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.addBatch("create table sg12 (tag1 string tag, s1 int32 field, s2 int32 field)"); + statement.addBatch("insert into sg12(tag1, time, s1) values('tag1', 3,1)"); + statement.addBatch("insert into sg12(tag1, time, s1) values('tag1', 3,1)"); + statement.addBatch("insert into sg12(tag1, time, s1) values('tag1', 1,1)"); + statement.addBatch("insert into sg12(tag1, time, s1) values('tag1', 2,1)"); + statement.addBatch("insert into sg12(tag1, time, s2) values('tag1', 2,2)"); + statement.addBatch("insert into sg12(tag1, time, s2) values('tag1', 1,2)"); + statement.addBatch("insert into sg12(tag1, time, s2) values('tag1', 3,2)"); + statement.addBatch("insert into sg12(tag1, time, s3) values('tag1', 1,3)"); + statement.addBatch("insert into sg12(tag1, time, s3) values('tag1', 3,3)"); + statement.executeBatch(); + + try (ResultSet resultSet = + statement.executeQuery("select count(s1), count(s2), count(s3) from sg12")) { + + assertTrue(resultSet.next()); + assertEquals(3, resultSet.getInt(1)); + assertEquals(3, resultSet.getInt(2)); + assertEquals(2, resultSet.getInt(3)); + + assertFalse(resultSet.next()); + } + + statement.execute("flush"); + try (ResultSet resultSet = + statement.executeQuery("select count(s1), count(s2), count(s3) from sg12")) { + + assertTrue(resultSet.next()); + assertEquals(3, resultSet.getInt(1)); + assertEquals(3, resultSet.getInt(2)); + assertEquals(2, resultSet.getInt(3)); + + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testInsertAlignedWithEmptyPage3() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute( + "create table dev13 (tag1 string tag, s1 int32 field, s2 int32 field, s3 int32 field)"); + for (int i = 0; i < 100; i++) { + if (i >= 49) { + statement.addBatch( + "insert into dev13(tag1,time,s1,s2,s3) values(" + + "\'GPS\'" + + "," + + i + + "," + + i + + "," + + i + + "," + + i + + ")"); + } else { + statement.addBatch( + "insert into dev13(tag1,time,s1,s2) values(" + + "\'GPS\'" + + "," + + i + + "," + + i + + "," + + i + + ")"); + } + } + statement.executeBatch(); + statement.execute("flush"); + int rowCount = 0; + try (ResultSet resultSet = statement.executeQuery("select s3 from dev13")) { + while (resultSet.next()) { + int v = resultSet.getInt(1); + if (rowCount >= 49) { + assertEquals(rowCount, v); + } else { + assertTrue(resultSet.wasNull()); + } + rowCount++; + } + assertEquals(100, rowCount); + } + + try (ResultSet resultSet = statement.executeQuery("select s2 from dev13")) { + rowCount = 0; + while (resultSet.next()) { + assertEquals(rowCount, resultSet.getInt(1)); + rowCount++; + } + assertEquals(100, rowCount); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from dev13")) { + rowCount = 0; + while (resultSet.next()) { + assertEquals(rowCount, resultSet.getInt(1)); + rowCount++; + } + assertEquals(100, rowCount); + } + } + } + + @Test + public void testExtendTextColumn() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"t1\""); + statement.execute("create table sg14 (tag1 string tag, s1 string field, s2 string field)"); + statement.execute("insert into sg14(tag1,time,s1,s2) values('d1',1,'test','test')"); + statement.execute("insert into sg14(tag1,time,s1,s2) values('d1',2,'test','test')"); + statement.execute("insert into sg14(tag1,time,s1,s2) values('d1',3,'test','test')"); + statement.execute("insert into sg14(tag1,time,s1,s2) values('d1',4,'test','test')"); + statement.execute("insert into sg14(tag1,time,s1,s3) values('d1',5,'test','test')"); + statement.execute("insert into sg14(tag1,time,s1,s2) values('d1',6,'test','test')"); + statement.execute("flush"); + statement.execute("insert into sg14(tag1,time,s1,s3) values('d1',7,'test','test')"); + fail(); + } catch (SQLException ignored) { + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java new file mode 100644 index 0000000000000..47e3a4b5e84db --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -0,0 +1,1197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.nio.charset.Charset; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showRegionColumnHeaders; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBInsertTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getDataNodeCommonConfig() + .setWriteMemoryProportion("10000000:1"); + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database if not exists test"); + statement.execute("use test"); + statement.execute( + "CREATE TABLE sg10(tag1 string tag, s1 int64 field, s2 float field, s3 string field)"); + statement.execute( + "CREATE TABLE sg11(tag1 string tag, s1 int64 field, s2 float field, s3 string field)"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInsertMultiPartition() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("create table sg1 (tag1 string tag, s1 int32 field)"); + statement.execute("insert into sg1(tag1,time,s1) values('d1',1,2)"); + statement.execute("flush"); + statement.execute("insert into sg1(tag1,time,s1) values('d1',2,2)"); + statement.execute("insert into sg1(tag1,time,s1) values('d1',604800001,2)"); + statement.execute("flush"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testShowRegion() { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("create table sg2 (tag1 string tag, s1 int32 field)"); + statement.execute("insert into sg2(tag1,time,s1) values('d1',1,2)"); + statement.execute("flush"); + statement.execute("insert into sg2(tag1,time,s1) values('d1',2,2)"); + statement.execute("insert into sg2(tag1,time,s1) values('d1',604800001,2)"); + statement.execute("flush"); + + // Test show regions in table model + try (final ResultSet resultSet = statement.executeQuery("show regions")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showRegionColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showRegionColumnHeaders.size(); i++) { + assertEquals( + showRegionColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + assertTrue(resultSet.next()); + } + + try (final ResultSet resultSet = statement.executeQuery("show regions from test")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showRegionColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showRegionColumnHeaders.size(); i++) { + assertEquals( + showRegionColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + assertTrue(resultSet.next()); + } + } catch (final Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testInsertTimeAtAnyIndex() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.addBatch( + "create table IF NOT EXISTS db2(tag1 string tag, s1 int32 field, s2 int32 field)"); + statement.addBatch("insert into db2(tag1, s1, s2, time) values ('d1', 2, 3, 1)"); + statement.addBatch("insert into db2(tag1, s1, time, s2) values ('d1', 20, 10, 30)"); + statement.addBatch("insert into db2(tag1, \"time\", s1, s2) values ('d1', 100, 200, 300)"); + statement.executeBatch(); + + try (ResultSet resultSet = statement.executeQuery("select time, s1 from db2")) { + assertTrue(resultSet.next()); + assertEquals(1, resultSet.getLong(1)); + assertEquals(2, resultSet.getInt(2)); + assertTrue(resultSet.next()); + assertEquals(10, resultSet.getLong(1)); + assertEquals(20, resultSet.getInt(2)); + assertTrue(resultSet.next()); + assertEquals(100, resultSet.getLong(1)); + assertEquals(200, resultSet.getInt(2)); + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testInsertMultiTime() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + try { + statement.addBatch("use \"test\""); + statement.addBatch("create table t3(tag1 string tag, s1 int32 field, s2 int32 field)"); + statement.addBatch("insert into t3(tag1, s1, s2, time, time) values ('d1', 2, 3, 1, 1)"); + statement.executeBatch(); + fail(); + } catch (SQLException e) { + // expected + } + + } catch (SQLException e) { + fail(); + } + } + + @Test + public void testPartialInsertionAllFailed() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + try { + statement.execute("USE \"test\""); + statement.execute("create table sg4 (tag1 string tag, s0 int32 field)"); + statement.execute("INSERT INTO sg4(tag1, timestamp, s0) VALUES ('tag', 1, 1)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Unknown column category")); + } + } + } + + @Test + public void testPartialInsertTablet() { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use \"test\""); + session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + session.executeNonQueryStatement( + "create table sg6 (tag1 string tag, s1 int64 field, s2 int64 field)"); + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + "sg6", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 300); + long timestamp = 0; + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 4; s++) { + long value = timestamp; + if (s == 0) { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, "d1"); + } else { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + } + timestamp++; + } + timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 4; s++) { + long value = timestamp; + if (s == 0) { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, "d1"); + } else { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + } + timestamp++; + } + try { + session.insert(tablet); + } catch (Exception e) { + if (!e.getMessage().contains("507")) { + fail(e.getMessage()); + } + } finally { + session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + } + try (SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM sg6")) { + assertEquals(dataSet.getColumnNames().size(), 4); + assertEquals(dataSet.getColumnNames().get(0), "time"); + assertEquals(dataSet.getColumnNames().get(1), "tag1"); + assertEquals(dataSet.getColumnNames().get(2), "s1"); + assertEquals(dataSet.getColumnNames().get(3), "s2"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long time = rowRecord.getFields().get(0).getLongV(); + assertEquals(time, rowRecord.getFields().get(2).getLongV()); + assertEquals(time, rowRecord.getFields().get(3).getLongV()); + cnt++; + } + Assert.assertEquals(200, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testInsertNull() { + String[] retArray = + new String[] { + "1,d2,null,1.0,1,", "2,d2,true,null,2,", "3,d2,true,3.0,null,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute( + "CREATE TABLE sg7 (tag1 string tag, s1 boolean field, s2 float field, s3 int32 field)"); + statement.execute("insert into sg7(tag1,time,s1,s2,s3) values('d2',1,null,1.0,1)"); + statement.execute("insert into sg7(tag1,time,s1,s2,s3) values('d2',2,true,null,2)"); + statement.execute("insert into sg7(tag1,time,s1,s2,s3) values('d2',3,true,3.0,null)"); + + try (ResultSet resultSet = statement.executeQuery("select * from sg7")) { + assertNotNull(resultSet); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,tag1,s1,s2,s3", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.INTEGER, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + if (i == 1) { + actualBuilder.append(resultSet.getTimestamp(i).getTime()).append(","); + } else { + actualBuilder.append(resultSet.getString(i)).append(","); + } + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + assertEquals(3, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testInsertNaN() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("CREATE TABLE sg8 (tag1 string tag, s1 float field, s2 double field)"); + // NaN should be a string literal, i.e., 'NaN', not NaN or "NaN" + try { + statement.execute("insert into sg8(tag1,time,s1,s2) values('d2',1,NaN,NaN)"); + fail("expected exception"); + } catch (SQLException e) { + assertEquals( + "701: Cannot insert identifier NaN, please use string literal", e.getMessage()); + } + try { + statement.execute("insert into sg8(tag1,time,s1,s2) values('d2',1,\"NaN\",\"NaN\")"); + fail("expected exception"); + } catch (SQLException e) { + assertEquals( + "701: Cannot insert identifier \"NaN\", please use string literal", e.getMessage()); + } + + statement.execute("insert into sg8(tag1,time,s1,s2) values('d2',1,'NaN','NaN')"); + + try (ResultSet resultSet = statement.executeQuery("select * from sg8")) { + assertNotNull(resultSet); + int cnt = 0; + while (resultSet.next()) { + assertEquals(1, resultSet.getLong("time")); + assertTrue(Float.isNaN(resultSet.getFloat("s1"))); + assertTrue(Double.isNaN(resultSet.getDouble("s2"))); + cnt++; + } + assertEquals(1, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // aggregation + @Test + public void testInsertWithoutTime() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE \"test\""); + statement.execute( + "CREATE TABLE sg9(tag1 string tag, s1 int64 field, s2 float field, s3 string field)"); + statement.execute("insert into sg9(tag1, s1, s2, s3) values ('d1',1, 1, '1')"); + Thread.sleep(1); + statement.execute("insert into sg9(tag1, s2, s1, s3) values ('d1',2, 2, '2')"); + Thread.sleep(1); + statement.execute("insert into sg9(tag1, s3, s2, s1) values ('d1','3', 3, 3)"); + Thread.sleep(1); + statement.execute("insert into sg9(tag1, s1) values ('d1',1)"); + statement.execute("insert into sg9(tag1, s2) values ('d1',2)"); + statement.execute("insert into sg9(tag1, s3) values ('d1','3')"); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2"}; + String[] retArray = new String[] {"4,4,4,"}; + tableResultSetEqualTest( + "select count(s1), count(s2), count(s3) from sg9", expectedHeader, retArray, "test"); + } + + @Test + public void testInsertMultiRow() { + assertTableNonQueryTestFail( + "insert into sg10(s3) values ('d1', '1'), ('d1', '2')", + "need timestamps when insert multi rows", + "test"); + assertTableNonQueryTestFail( + "insert into sg10(tag1, s1, s2) values ('d1', 1, 1), ('d1', 2, 2)", + "need timestamps when insert multi rows", + "test"); + } + + @Test + public void testInsertWithMultiTimesColumns() { + assertTableNonQueryTestFail( + "insert into sg11(tag1, time, time) values ('d1', 1, 1)", + "One row should only have one time value", + "test"); + assertTableNonQueryTestFail( + "insert into sg11(tag1, time, s1, time) values ('d1', 1, 1, 1)", + "One row should only have one time value", + "test"); + } + + // aggregation + @Test + public void testInsertMultiRow2() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + Statement st0 = connection.createStatement(); + st0.execute("use \"test\""); + st0.execute( + "create table wf12 (tag1 string tag, status boolean field, temperature float field)"); + st0.execute("insert into wf12(tag1, time, status) values ('wt01', 1, true)"); + st0.execute( + "insert into wf12(tag1, time, status) values ('wt01', 2, true), ('wt01', 3, false)"); + st0.execute( + "insert into wf12(tag1, time, status) values ('wt01', 4, true), ('wt01', 5, true), ('wt01', 6, false)"); + + st0.execute( + "insert into wf12(tag1, time, temperature, status) values ('wt01', 7, 15.3, true)"); + st0.execute( + "insert into wf12(tag1, time, temperature, status) values ('wt01', 8, 18.3, false), ('wt01', 9, 23.1, false)"); + st0.execute( + "insert into wf12(tag1, time, temperature, status) values ('wt01', 10, 22.3, true), ('wt01', 11, 18.8, false), ('wt01', 12, 24.4, true)"); + st0.close(); + + Statement st1 = connection.createStatement(); + ResultSet rs1 = st1.executeQuery("select count(status) from wf12"); + rs1.next(); + long countStatus = rs1.getLong(1); + assertEquals(countStatus, 12L); + + ResultSet rs2 = st1.executeQuery("select count(temperature) from wf12"); + rs2.next(); + long countTemperature = rs2.getLong(1); + assertEquals(countTemperature, 6L); + + st1.close(); + } + } + + @Test + public void testInsertMultiRowWithMisMatchDataType() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + try { + Statement st1 = connection.createStatement(); + st1.execute("use \"test\""); + st1.execute( + "create table wf13 (tag1 string tag, status boolean field, temperature float field)"); + st1.execute( + "insert into wf13(tag1, time, status) values('wt01', 1, 1.0), ('wt01', 2, 'hello')"); + fail(); + } catch (SQLException e) { + assertTrue( + e.getMessage().contains(Integer.toString(TSStatusCode.METADATA_ERROR.getStatusCode()))); + } + } + } + + @Test + public void testInsertMultiRowWithNull() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + st1.execute("use \"test\""); + st1.execute( + "create table wf14 (tag1 string tag, status boolean field, temperature float field)"); + st1.execute("insert into wt14(time, s1, s2) values(100, null, 1), (101, null, 2)"); + fail(); + } catch (SQLException e) { + assertEquals("550: Table 'test.wt14' does not exist.", e.getMessage()); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + try (Statement st2 = connection.createStatement()) { + st2.execute("use \"test\""); + st2.execute("CREATE TABLE wf15 (wt string tag, s1 double field, s2 double field)"); + st2.execute( + "INSERT INTO wf15(wt, time, s1) VALUES ('1', 6, 10),('1', 7,12),('1', 8,14),('1', 9,160),('1', 10,null),('1', 11,58)"); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + } + + @Test + public void testInsertMultiRowWithWrongTimestampPrecision() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + try { + st1.execute("use \"test\""); + st1.execute( + "insert into wf16(tag1, time, status) values('wt01', 1618283005586000, true), ('wt01', 1618283005586001, false)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); + } + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + try { + st1.execute("use \"test\""); + st1.execute( + "insert into wf16(tag1, time, status) values('wt01', -1618283005586000, true), ('wt01', -1618283005586001, false)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); + } + } + } + + @Test + public void testInsertMultiRowWithMultiTimePartition() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + st1.execute("use \"test\""); + st1.execute("create table sg17 (tag1 string tag, s1 int32 field)"); + st1.execute("insert into sg17(tag1, time, s1) values('d1', 604800010,1)"); + st1.execute("flush"); + st1.execute( + "insert into sg17(tag1, time, s1) values('d1', 604799990,1), ('d1', 604800001,1)"); + st1.execute("flush"); + + ResultSet rs1 = st1.executeQuery("select time, s1 from sg17"); + assertTrue(rs1.next()); + assertEquals(604799990, rs1.getLong("time")); + assertTrue(rs1.next()); + assertEquals(604800001, rs1.getLong("time")); + assertTrue(rs1.next()); + assertEquals(604800010, rs1.getLong("time")); + assertFalse(rs1.next()); + } + } + + @Test + public void testInsertAttributes() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + st1.execute("use \"test\""); + st1.execute( + "create table if not exists sg18 (tag1 string tag, s1 string attribute, s2 int32 field)"); + st1.execute("insert into sg18(tag1, s1, s2) values('d1','1', 1)"); + st1.execute("insert into sg18(tag1, s1, s2) values('d2', 2, 2)"); + + ResultSet rs1 = st1.executeQuery("select time, s1, s2 from sg18 order by s1"); + assertTrue(rs1.next()); + assertEquals("1", rs1.getString("s1")); + assertTrue(rs1.next()); + assertEquals("2", rs1.getString("s1")); + assertFalse(rs1.next()); + } + } + + @Test + public void testInsertCaseSensitivity() + throws SQLException, IoTDBConnectionException, StatementExecutionException { + // column case sensitivity + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + st1.execute("use \"test\""); + st1.execute( + "create table if not exists sg19 (tag1 string tag, ss1 string attribute, ss2 int32 field)"); + // lower case + st1.execute("insert into sg19(time, tag1, ss1, ss2) values(1, 'd1','1', 1)"); + st1.execute("insert into sg19(time, tag1, ss1, ss2) values(2, 'd2', 2, 2)"); + // upper case + st1.execute("insert into sg19(TIME, TAG1, SS1, SS2) values(3, 'd3','3', 3)"); + st1.execute("insert into sg19(TIME, TAG1, SS1, SS2) values(4, 'd4', 4, 4)"); + // mixed + st1.execute("insert into sg19(TIme, Tag1, Ss1, Ss2) values(5, 'd5','5', 5)"); + st1.execute("insert into sg19(TIme, Tag1, sS1, sS2) values(6, 'd6', 6, 6)"); + + ResultSet rs1 = st1.executeQuery("select time, ss1, ss2 from sg19 order by time"); + for (int i = 1; i <= 6; i++) { + assertTrue(rs1.next()); + assertEquals(i, rs1.getLong("time")); + assertEquals(String.valueOf(i), rs1.getString("ss1")); + assertEquals(i, rs1.getInt("ss2")); + } + assertFalse(rs1.next()); + } + + // table case sensitivity with record and auto creation + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"test\""); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + + Tablet tablet = + new Tablet( + "TaBle19_2", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + } + session.insert(tablet); + tablet.reset(); + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_2 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + + // table case sensitivity with record and no auto creation + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "CREATE TABLE tAbLE19_3 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + List fieldIds = + schemaList.stream() + .map(IMeasurementSchema::getMeasurementName) + .collect(Collectors.toList()); + List dataTypes = IMeasurementSchema.getDataTypeList(schemaList); + + long timestamp = 0; + + Tablet tablet = + new Tablet( + "TaBle19_3", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + } + session.insert(tablet); + tablet.reset(); + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_3 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + + // table case sensitivity with tablet and no auto creation + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"test\""); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "TaBle19_4", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_4 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + + // table case sensitivity with tablet and auto creation + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "CREATE TABLE tAbLE19_5 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "TaBle19_5", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_5 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + } + + @Test + public void testInsertKeyword() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "create table table20 (" + + "device_id string tag," + + "attribute STRING ATTRIBUTE," + + "boolean boolean FIELD," + + "int32 int32 FIELD," + + "int64 int64 FIELD," + + "float float FIELD," + + "double double FIELD," + + "text text FIELD," + + "string string FIELD," + + "blob blob FIELD," + + "timestamp01 timestamp FIELD," + + "date date FIELD)"); + + List schemas = new ArrayList<>(); + schemas.add(new MeasurementSchema("device_id", TSDataType.STRING)); + schemas.add(new MeasurementSchema("attribute", TSDataType.STRING)); + schemas.add(new MeasurementSchema("boolean", TSDataType.BOOLEAN)); + schemas.add(new MeasurementSchema("int32", TSDataType.INT32)); + schemas.add(new MeasurementSchema("int64", TSDataType.INT64)); + schemas.add(new MeasurementSchema("float", TSDataType.FLOAT)); + schemas.add(new MeasurementSchema("double", TSDataType.DOUBLE)); + schemas.add(new MeasurementSchema("text", TSDataType.TEXT)); + schemas.add(new MeasurementSchema("string", TSDataType.STRING)); + schemas.add(new MeasurementSchema("blob", TSDataType.BLOB)); + schemas.add(new MeasurementSchema("timestamp", TSDataType.TIMESTAMP)); + schemas.add(new MeasurementSchema("date", TSDataType.DATE)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table20", + IMeasurementSchema.getMeasurementNameList(schemas), + IMeasurementSchema.getDataTypeList(schemas), + columnTypes, + 10); + + for (long row = 0; row < 10; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("device_id", rowIndex, "1"); + tablet.addValue("attribute", rowIndex, "1"); + tablet.addValue("boolean", rowIndex, true); + tablet.addValue("int32", rowIndex, Integer.valueOf("1")); + tablet.addValue("int64", rowIndex, Long.valueOf("1")); + tablet.addValue("float", rowIndex, Float.valueOf("1.0")); + tablet.addValue("double", rowIndex, Double.valueOf("1.0")); + tablet.addValue("text", rowIndex, "true"); + tablet.addValue("string", rowIndex, "true"); + tablet.addValue("blob", rowIndex, new Binary("iotdb", Charset.defaultCharset())); + tablet.addValue("timestamp", rowIndex, 1L); + tablet.addValue("date", rowIndex, LocalDate.parse("2024-08-15")); + } + session.insert(tablet); + + SessionDataSet rs1 = + session.executeQueryStatement( + "select time, device_id, attribute, boolean, int32, int64, float, double, text, string, blob, timestamp, date from table20 order by time"); + for (int i = 0; i < 10; i++) { + RowRecord rec = rs1.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + assertEquals("1", rec.getFields().get(1).getStringValue()); + assertEquals("1", rec.getFields().get(2).getStringValue()); + assertTrue(rec.getFields().get(3).getBoolV()); + assertEquals(1, rec.getFields().get(4).getIntV()); + assertEquals(1, rec.getFields().get(5).getLongV()); + assertEquals(1.0, rec.getFields().get(6).getFloatV(), 0.001); + assertEquals(1.0, rec.getFields().get(7).getDoubleV(), 0.001); + assertEquals("true", rec.getFields().get(8).getStringValue()); + assertEquals("true", rec.getFields().get(9).getStringValue()); + assertEquals("0x696f746462", rec.getFields().get(10).getStringValue()); + assertEquals(1, rec.getFields().get(11).getLongV()); + assertEquals("20240815", rec.getFields().get(12).getStringValue()); + } + assertFalse(rs1.hasNext()); + } + } + + @Test + public void testInsertSingleColumn() throws SQLException, InterruptedException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + st1.execute("use \"test\""); + st1.execute( + "create table if not exists sg21 (tag1 string tag, ss1 string attribute, ss2 int32 field)"); + // only tag + try { + st1.execute("insert into sg21(tag1) values('1')"); + } catch (SQLException e) { + assertEquals("507: No Field column present, please check the request", e.getMessage()); + } + // only time + try { + st1.execute("insert into sg21(time) values(1)"); + } catch (SQLException e) { + assertEquals( + "507: No column other than Time present, please check the request", e.getMessage()); + } + // sleep a while to avoid the same timestamp between two insertions + Thread.sleep(10); + // only attribute + try { + st1.execute("insert into sg21(ss1) values('1')"); + } catch (SQLException e) { + assertEquals("507: No Field column present, please check the request", e.getMessage()); + } + // sleep a while to avoid the same timestamp between two insertions + Thread.sleep(10); + // only field + st1.execute("insert into sg21(ss2) values(1)"); + + ResultSet rs1 = st1.executeQuery("show devices from sg21"); + assertTrue(rs1.next()); + // from "insert into sg21(ss2) values(1)" + assertEquals(null, rs1.getString("tag1")); + assertFalse(rs1.next()); + // from "insert into sg21(tag1) values('1')" + assertEquals(null, rs1.getString("tag1")); + assertFalse(rs1.next()); + + rs1 = st1.executeQuery("select time, ss1, ss2 from sg21 order by time"); + assertTrue(rs1.next()); + rs1.getString("ss1"); + assertTrue(rs1.wasNull()); + rs1.getInt("ss2"); + assertEquals(1, rs1.getInt("ss2")); + + assertFalse(rs1.next()); + } + } + + @Test + public void testInsertWithTTL() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("create table sg22 (tag1 string tag, s1 int64 field)"); + statement.execute("alter table sg22 set properties TTL=1"); + statement.execute( + String.format( + "insert into sg22(tag1,time,s1) values('d1',%s,2)", + System.currentTimeMillis() - 10000)); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("less than ttl time bound")); + } + } + + @Test + public void testInsertTabletWithTTL() + throws IoTDBConnectionException, StatementExecutionException { + long ttl = 1; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use \"test\""); + session.executeNonQueryStatement("create table sg23 (tag1 string tag, s1 int64 field)"); + session.executeNonQueryStatement("alter table sg23 set properties TTL=" + ttl); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + + // all expired + long timestamp = 0; + Tablet tablet = + new Tablet( + "sg23", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 3; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("s1", rowIndex, row); + } + try { + session.insert(tablet); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("less than ttl time bound")); + } + + // partial expired + tablet.reset(); + timestamp = System.currentTimeMillis() - 10000; + for (long row = 0; row < 4; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("s1", rowIndex, row); + timestamp += 10000; + } + + try { + session.insert(tablet); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("less than ttl time bound")); + } + + // part of data is indeed inserted + long timeLowerBound = System.currentTimeMillis() - ttl; + SessionDataSet dataSet = session.executeQueryStatement("select time, s1 from sg23"); + int count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + Assert.assertTrue(record.getFields().get(0).getLongV() > timeLowerBound); + count++; + } + Assert.assertEquals(2, count); + } + } + + @Test + public void testInsertUnsequenceData() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"test\""); + // the table is missing column "m2" + session.executeNonQueryStatement( + "CREATE TABLE table4 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + // the insertion contains "m2" + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("m2", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table4", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + tablet.addValue("m2", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insert(tablet); + } catch (StatementExecutionException e) { + // a partial insertion should be reported + if (!e.getMessage() + .equals( + "507: Fail to insert fields [m2] caused by [Column m2 does not exists or fails to be created]")) { + throw e; + } + } + tablet.reset(); + } + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 14 - row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + tablet.addValue("m2", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insert(tablet); + } catch (StatementExecutionException e) { + if (!e.getMessage() + .equals( + "507: Fail to insert fields [m2] caused by [Column m2 does not exists or fails to be created]")) { + throw e; + } + } + tablet.reset(); + } + } + session.executeNonQueryStatement("FLush"); + + int cnt = 0; + SessionDataSet dataSet = session.executeQueryStatement("select * from table4"); + while (dataSet.hasNext()) { + dataSet.next(); + cnt++; + } + assertEquals(29, cnt); + } + } + + @Test + public void testInsertAllNullRow() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + st1.execute("use \"test\""); + st1.execute("create table table5(d1 string tag, s1 int32 field, s2 int32 field)"); + + st1.execute("insert into table5(time, d1,s1,s2) values(1,'a',1,null)"); + // insert all null row + st1.execute("insert into table5(time, d1,s1,s2) values(2,'a',null,null)"); + + ResultSet rs1 = st1.executeQuery("select * from table5"); + assertTrue(rs1.next()); + assertEquals("1", rs1.getString("s1")); + assertNull(rs1.getString("s2")); + assertTrue(rs1.next()); + assertNull(rs1.getString("s1")); + assertNull(rs1.getString("s2")); + assertFalse(rs1.next()); + + st1.execute("flush"); + + rs1 = st1.executeQuery("select * from table5"); + assertTrue(rs1.next()); + assertEquals("1", rs1.getString("s1")); + assertNull(rs1.getString("s2")); + assertTrue(rs1.next()); + assertNull(rs1.getString("s1")); + assertNull(rs1.getString("s2")); + assertFalse(rs1.next()); + } + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + assertNotNull(typeIndex); + assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java new file mode 100644 index 0000000000000..53e8bc9487e33 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.sql.Connection; +import java.sql.Statement; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBLoadConfigurationTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadConfiguration() throws IOException { + DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + String confPath = + dataNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + "iotdb-system.properties"; + long length = new File(confPath).length(); + try (FileWriter fileWriter = new FileWriter(confPath, true)) { + fileWriter.write(System.lineSeparator()); + fileWriter.write("target_compaction_file_size=t"); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("LOAD CONFIGURATION"); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("NumberFormatException")); + } finally { + try (FileChannel fileChannel = + FileChannel.open(new File(confPath).toPath(), StandardOpenOption.WRITE)) { + fileChannel.truncate(length); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java new file mode 100644 index 0000000000000..16d92c213436d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Notice that, all test begins with "IoTDB" is integration test. All test which will start the + * IoTDB server should be defined as integration test. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBMultiDeviceTableIT { + + @BeforeClass + public static void setUp() throws Exception { + // use small page + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setMaxNumberOfPointsInPage(100) + .setPageSizeInByte(1024 * 15) + .setGroupSizeInByte(1024 * 100) + .setMemtableSizeThreshold(1024 * 100) + .setPartitionInterval(100) + .setQueryThreadCount(2) + .setCompressor("LZ4"); + + EnvFactory.getEnv().initClusterEnvironment(); + + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.addBatch("CREATE DATABASE test"); + statement.addBatch("USE \"test\""); + statement.addBatch( + "create table t (tag1 string tag, tag2 string tag, s0 int32 field, s1 int32 field)"); + + // insert of data time range :0-100 into fans + for (int time = 0; time < 100; time++) { + + String sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d2',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d3',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d2',%s,%s)", time, time % 4); + statement.addBatch(sql); + } + + // insert large amount of data time range : 1370 ~ 2400 + for (int time = 1370; time < 2400; time++) { + + String sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d2',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d3',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d2',%s,%s)", time, time % 4); + statement.addBatch(sql); + } + + // insert large amount of data time range : 300 ~ 1360 + for (int time = 300; time < 1360; time++) { + // System.out.println("===" + time); + String sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d2',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d3',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d2',%s,%s)", time, time % 4); + statement.addBatch(sql); + } + + statement.addBatch("flush"); + + // unsequential data, memory data + for (int time = 1000; time < 1100; time++) { + + String sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d2',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d3',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d2',%s,%s)", time, time % 4); + statement.addBatch(sql); + } + + // sequential data, memory data + for (int time = 20000; time < 20100; time++) { + + String sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d2',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1,tag2,time,s0) values('fans','d3',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d0',%s,%s)", time, time % 7); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d1',%s,%s)", time, time % 4); + statement.addBatch(sql); + sql = + String.format( + "insert into t(tag1, tag2,time,s0) values('car','d2',%s,%s)", time, time % 4); + statement.addBatch(sql); + } + assertTrue(Arrays.stream(statement.executeBatch()).allMatch(i -> i == 200)); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testMultiDeviceQueryAndDelete() { + testSelectAll(); + // TODO: add support + // testSelectAfterDelete(); + } + + private void testSelectAll() { + String selectSql = "select * from t"; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + Map lastTimeMap = new HashMap<>(); + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + int cnt = 0; + while (resultSet.next()) { + String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); + long before = lastTimeMap.getOrDefault(tag, -1L); + long cur = resultSet.getTimestamp("time").getTime(); + if (cur <= before) { + fail("time order wrong!"); + } + lastTimeMap.put(tag, cur); + cnt++; + } + assertEquals(16030, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void testSelectAfterDelete() { + String selectSql = "select * from t"; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE \"test\""); + + statement.execute("DELETE FROM t WHERE tag1='fans' and time <= 100"); + statement.execute("DELETE FROM t WHERE tag1='car' and time <= 100"); + statement.execute("DELETE FROM t WHERE tag1='fans' and time >= 20050 and time < 20100"); + statement.execute("DELETE FROM t WHERE tag1='car' and time >= 20050 and time < 20100"); + + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + int cnt = 0; + long before = -1; + while (resultSet.next()) { + long cur = Long.parseLong(resultSet.getString("time")); + if (cur <= before) { + fail("time order wrong!"); + } + before = cur; + cnt++; + } + assertEquals(2140, cnt); + } + + statement.execute("DELETE FROM t WHERE tag1 = 'fans' and time <= 20000"); + statement.execute("DELETE FROM t WHERE tag1 = 'car' and time <= 20000"); + + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + int cnt = 0; + long before = -1; + while (resultSet.next()) { + long cur = resultSet.getTimestamp("time").getTime(); + if (cur <= before) { + fail("time order wrong!"); + } + before = cur; + cnt++; + } + assertEquals(49, cnt); + } + + statement.execute("DELETE FROM t WHERE time >= 20000"); + + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + int cnt = 0; + long before = -1; + while (resultSet.next()) { + long cur = resultSet.getTimestamp("time").getTime(); + if (cur <= before) { + fail("time order wrong!"); + } + before = cur; + cnt++; + } + assertEquals(0, cnt); + } + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java new file mode 100644 index 0000000000000..1e80e36e42d99 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java @@ -0,0 +1,2813 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.Arrays; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.FULL; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.LEFT; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.RIGHT; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.ONLY_SUPPORT_EQUI_JOIN; +import static org.junit.Assert.fail; + +/** In this IT, table has more than one TAGs and Attributes. */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBMultiTAGsWithAttributesTableIT { + private static final String DATABASE_NAME = "db"; + + private static final String[] sql1 = + new String[] { + "CREATE DATABASE db", + "USE db", + "CREATE TABLE table0 (device string tag, level string tag, attr1 string attribute, attr2 string attribute, num int32 field, bigNum int64 field, " + + "floatNum FLOAT field, str TEXT field, bool BOOLEAN field, date DATE field, blob BLOB field, ts TIMESTAMP field, stringV STRING field, doubleNum DOUBLE field)", + "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l1', 'c', 'd', 0,3,2947483648,231.2121,'coconut',FALSE)", + "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool,blob,ts,doubleNum) values('d1', 'l2', 'y', 'z', 20,2,2147483648,434.12,'pineapple',TRUE,X'108DCD62',2024-09-24T06:15:35.000+00:00,6666.8)", + "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l3', 't', 'a', 40,1,2247483648,12.123,'apricot',TRUE)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l4', 80,9,2147483646,43.12,'apple',FALSE)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l5', 100,8,2147483964,4654.231,'papaya',TRUE)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l1', 31536000000,6,2147483650,1231.21,'banana',TRUE)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l2',31536000100,10,3147483648,231.55,'pumelo',FALSE)", + "insert into table0(device,level, time,num,bigNum,floatNum,str,bool,doubleNum) values('d1', 'l3',31536000500,4,2147493648,213.1,'peach',FALSE,6666.3)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool,blob,ts,stringV) values('d1', 'l4',31536001000,5,2149783648,56.32,'orange',FALSE,X'108DCD62',2024-09-15T06:15:35.000+00:00,'test-string1')", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool,blob,ts) values('d1', 'l5',31536010000,7,2147983648,213.112,'lemon',TRUE,X'108DCD63',2024-09-25T06:15:35.000+00:00)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l1',31536100000,11,2147468648,54.121,'pitaya',FALSE)", + "insert into table0(device,level,attr1,attr2,time,num,bigNum,floatNum,str,bool) values('d2','l1','d','c',0,3,2947483648,231.2121,'coconut',FALSE)", + "insert into table0(device,level,attr1,time,num,bigNum,floatNum,str,bool) values('d2','l2', 'vv', 31536000100,10,3147483648,231.55,'pumelo',FALSE)", + "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l2', 'yy', 'zz', 41536000000,12,2146483648,45.231,'strawberry',FALSE)", + "flush", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l3',41536000020,14,2907483648,231.34,'cherry',FALSE)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool) values('d1', 'l4',41536900000,13,2107483648,54.12,'lychee',TRUE)", + "insert into table0(device, level, time,num,bigNum,floatNum,str,bool,date) values('d1', 'l5',51536000000,15,3147483648,235.213,'watermelon',TRUE,'2022-01-01')" + }; + + private static final String[] sql2 = + new String[] { + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l2',20,2,2147483648,434.12,'pineapple',TRUE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l3',40,1,2247483648,12.123,'apricot',TRUE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool,blob,ts,stringV,doubleNum) values('d2','l4',80,9,2147483646,43.12,'apple',FALSE,X'108DCD63',2024-09-20T06:15:35.000+00:00,'test-string2',6666.7)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l5',100,8,2147483964,4654.231,'papaya',TRUE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l1',31536000000,6,2147483650,1231.21,'banana',TRUE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l3',31536000500,4,2147493648,213.1,'peach',FALSE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l4',31536001000,5,2149783648,56.32,'orange',FALSE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l5',31536010000,7,2147983648,213.112,'lemon',TRUE)", + "flush", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool,ts,StringV) values('d2','l1',31536100000,11,2147468648,54.121,'pitaya',FALSE,2024-08-01T06:15:35.000+00:00,'test-string3')", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l2',41536000000,12,2146483648,45.231,'strawberry',FALSE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool,doubleNum) values('d2','l3',41536000020,14,2907483648,231.34,'cherry',FALSE,6666.4)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool) values('d2','l4',41536900000,13,2107483648,54.12,'lychee',TRUE)", + "insert into table0(device,level,time,num,bigNum,floatNum,str,bool,date) values('d2','l5',51536000000,15,3147483648,235.213,'watermelon',TRUE,'2023-01-01')" + }; + + private static final String[] sql3 = + new String[] { + "CREATE TABLE table1 (device string tag, level string tag, attr1 string attribute, attr2 string attribute, num int32 field, bigNum int64 field, " + + "floatNum double field, str TEXT field, bool BOOLEAN field, date DATE field, blob BLOB field, doubleNum DOUBLE field)", + "insert into table1(device, level, attr1, attr2, time, num, bigNum, floatNum, str, bool, date, blob) values('d1', 'l1', 'c', 'd', 0, 3, 2947483648,231.2121, 'coconut', FALSE, '2023-01-01', X'100DCD62')", + "insert into table1(device, level, attr1, attr2, time, num, bigNum, floatNum, str, bool, blob) values('d1', 'l2', 'y', 'z', 20, 2, 2147483648,434.12, 'pineapple', TRUE, X'101DCD62')", + "insert into table1(device, level, attr1, attr2, time, num, bigNum, floatNum, str, bool, blob) values('d1', 'l3', 't', 'a', 40, 1, 2247483648,12.123, 'apricot', TRUE, null)", + "insert into table1(device, level, time, num, bigNum, floatNum, str, bool, blob) values('d1', 'l4', 80, 9, 2147483646, 43.12, 'apple', FALSE, X'104DCD62')", + "insert into table1(device, level, time, num, bigNum, floatNum, str, bool, date, blob) values('d1', 'l5', 100, 8, 2147483964, 4654.231, 'papaya', TRUE, '2023-05-01', null)", + "flush", + "insert into table1(device, level, time, num, bigNum, floatNum, str, bool, blob) values('d1', 'l1', 31536000000, 6, 2147483650, 1231.21, 'banana', TRUE, X'106DCD62')", + "insert into table1(device, time, num, bigNum, floatNum, str, bool, blob) values('d999', 31536000000, 6, 2147483650, 1231.21, 'banana', TRUE, X'107DCD62')", + "insert into table1(level, time, num, bigNum, floatNum, str, bool, date, blob) values('l999', 31536000000, 6, 2147483650, 1231.21, 'banana', TRUE, '2023-10-01', X'108DCD62')", + "flush", + "insert into table1(time, device, level, attr1, attr2, num,bigNum,floatNum,str,bool) values(0, 'd11', 'l11', 'c', 'd', 3, 2947483648, 231.2121, 'coconut', FALSE)", + "flush", + "insert into table1(time, device, level, attr1, attr2, num,bigNum,floatNum,str,bool) values(10, 'd11', 'l11', 'c', 'd', 3, 2947483648, 231.2121, 'coconut', FALSE)", + "flush", + "insert into table1(time, device, level, attr1, attr2, num,bigNum,floatNum,str,bool) values(20, 'd11', 'l11', 'c', 'd', 3, 2947483648, 231.2121, 'coconut', FALSE)", + "flush", + "insert into table1(time, device, level, attr1, attr2, num,bigNum,floatNum,str,bool) values(30, 'd11', 'l11', 'c', 'd', 3, 2947483648, 231.2121, 'coconut', FALSE)", + "flush", + "insert into table1(time, device, level, attr1, attr2, num,bigNum,floatNum,str,bool) values(40, 'd11', 'l11', 'c', 'd', 3, 2947483648, 231.2121, 'coconut', FALSE)" + }; + + private static final String[] sql4 = + new String[] { + "create table students(region STRING TAG, student_tag INT32 FIELD, name STRING FIELD, genders text FIELD, date_of_birth DATE FIELD)", + "create table teachers(region STRING TAG, teacher_tag INT32 FIELD, course_tag INT32 FIELD, age INT32 FIELD)", + "create table courses(course_tag STRING TAG, course_name STRING FIELD, teacher_tag INT32 FIELD)", + "create table grades(grade_tag STRING TAG, course_tag INT32 FIELD, student_tag INT32 FIELD, score INT32 FIELD)", + "insert into students(time,region,student_tag,name,genders,date_of_birth) values" + + "(1,'haidian',1,'Lucy','女','2015-10-10'),(2,'haidian',2,'Jack','男','2015-09-24'),(3,'chaoyang',3,'Sam','男','2014-07-20'),(4,'chaoyang',4,'Lily','女','2015-03-28')," + + "(5,'xicheng',5,'Helen','女','2016-01-22'),(6,'changping',6,'Nancy','女','2017-12-20'),(7,'changping',7,'Mike','男','2016-11-22'),(8,'shunyi',8,'Bob','男','2016-05-12')", + "insert into teachers(time,region,teacher_tag,course_tag,age) values" + + "(1,'haidian',1001,10000001,25),(2,'haidian',1002,10000002,26),(3,'chaoyang',1003,10000003,28)," + + "(4,'chaoyang',1004,10000004,27),(5,'xicheng',1005,10000005,26)", + "insert into courses(time,course_tag,course_name,teacher_tag) values" + + "(1,10000001,'数学',1001),(2,10000002,'语文',1002),(3,10000003,'英语',1003)," + + "(4,10000004,'体育',1004),(5,10000005,'历史',1005)", + "insert into grades(time,grade_tag,course_tag,student_tag,score) values" + + "(1,1111,10000001,1,99),(2,1112,10000002,2,90),(3,1113,10000003,3,85),(4,1114,10000004,4,89),(5,1115,10000005,5,98)," + + "(6,1113,10000003,6,55),(7,1114,10000004,7,60),(8,1115,10000005,8,100),(9,1114,10000001,2,99),(10,1115,10000002,1,95)" + }; + + private static final String[] sql5 = + new String[] { + "create table tableA(device STRING TAG, value INT32 FIELD)", + "create table tableB(device STRING TAG, value INT32 FIELD)", + "create table tableC(device STRING TAG, value INT32 FIELD)", + "insert into tableA(time,device,value) values('2020-01-01 00:00:01.000', 'd1', 1)", + "insert into tableA(time,device,value) values('2020-01-01 00:00:03.000', 'd1', 3)", + "flush", + "insert into tableA(time,device,value) values('2020-01-01 00:00:05.000', 'd2', 5)", + "insert into tableA(time,device,value) values('2020-01-01 00:00:07.000', 'd2', 7)", + "flush", + "insert into tableB(time,device,value) values('2020-01-01 00:00:02.000', 'd1', 20)", + "insert into tableB(time,device,value) values('2020-01-01 00:00:03.000', 'd1', 30)", + "flush", + "insert into tableB(time,device,value) values('2020-01-01 00:00:03.000', 'd333', 333)", + "flush", + "insert into tableB(time,device,value) values('2020-01-01 00:00:04.000', 'd2', 40)", + "insert into tableB(time,device,value) values('2020-01-01 00:00:05.000', 'd2', 50)" + }; + + String[] expectedHeader; + String[] retArray; + static String sql; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); + EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setMaxTsBlockLineNumber(2) + .setMaxNumberOfPointsInPage(5); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String[] sqlList : Arrays.asList(sql1, sql2, sql3, sql4, sql5)) { + for (String sql : sqlList) { + statement.execute(sql); + } + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void onlyQueryAttribute2Test() { + String[] expectedHeader = + new String[] { + "time", "device", "attr2", "num", "str", + }; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,null,2,pineapple,", + "1970-01-01T00:00:00.040Z,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,d2,null,1,apricot,", + "1970-01-01T00:00:00.080Z,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,null,9,apple,", + "1970-01-01T00:00:00.100Z,d1,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d2,null,8,papaya,", + "1971-01-01T00:00:00.000Z,d1,d,6,banana,", + "1971-01-01T00:00:00.000Z,d2,c,6,banana,", + "1971-01-01T00:00:00.100Z,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,null,5,orange,", + "1971-01-01T00:00:10.000Z,d1,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d2,null,7,lemon,", + "1971-01-01T00:01:40.000Z,d1,d,11,pitaya,", + "1971-01-01T00:01:40.000Z,d2,c,11,pitaya,", + "1971-04-26T17:46:40.000Z,d1,zz,12,strawberry,", + "1971-04-26T17:46:40.000Z,d2,null,12,strawberry,", + "1971-04-26T17:46:40.020Z,d1,a,14,cherry,", + "1971-04-26T17:46:40.020Z,d2,null,14,cherry,", + "1971-04-26T18:01:40.000Z,d1,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d2,null,13,lychee,", + "1971-08-20T11:33:20.000Z,d1,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d2,null,15,watermelon,", + }; + tableResultSetEqualTest( + "select time, device, attr2, num, str from table0 order by time, device", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "num", "str", "attr2", + }; + retArray = + new String[] { + "3,coconut,d,", + "6,banana,d,", + "11,pitaya,d,", + "2,pineapple,zz,", + "10,pumelo,zz,", + "12,strawberry,zz,", + "1,apricot,a,", + "4,peach,a,", + "14,cherry,a,", + "9,apple,null,", + "5,orange,null,", + "13,lychee,null,", + "8,papaya,null,", + "7,lemon,null,", + "15,watermelon,null,", + "3,coconut,c,", + "6,banana,c,", + "11,pitaya,c,", + "2,pineapple,null,", + "10,pumelo,null,", + "12,strawberry,null,", + "1,apricot,null,", + "4,peach,null,", + "14,cherry,null,", + "9,apple,null,", + "5,orange,null,", + "13,lychee,null,", + "8,papaya,null,", + "7,lemon,null,", + "15,watermelon,null,", + }; + tableResultSetEqualTest( + "select num, str, attr2 from table0 order by device, level, time", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "time", "device", "num", "str", + }; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,3,coconut,", + "1971-01-01T00:00:00.000Z,d1,6,banana,", + "1971-01-01T00:01:40.000Z,d1,11,pitaya,", + }; + tableResultSetEqualTest( + "select time, device, num, str from table0 where attr2 = 'd' order by time,device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sortAttributeTest() { + String[] expectedHeader = + new String[] { + "time", "device", "level", "attr1", "attr2", "num", "bignum", "floatnum", "str", "bool" + }; + String[] retArray = + new String[] { + "1971-01-01T00:01:40.000Z,d2,l1,d,c,11,2147468648,54.121,pitaya,false,", + "1970-01-01T00:00:00.040Z,d1,l3,t,a,1,2247483648,12.123,apricot,true,", + "1971-01-01T00:00:00.500Z,d1,l3,t,a,4,2147493648,213.1,peach,false,", + "1971-04-26T17:46:40.020Z,d1,l3,t,a,14,2907483648,231.34,cherry,false,", + "1970-01-01T00:00:00.020Z,d2,l2,vv,null,2,2147483648,434.12,pineapple,true," + }; + tableResultSetEqualTest( + "select \"time\", \"device\", \"level\", \"attr1\", \"attr2\", \"num\", \"bignum\", \"floatnum\", \"str\", \"bool\" from table0 order by attr1, level desc, time asc offset 5 limit 5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sortAllTAGTimesTest() { + String[] expectedHeader = new String[] {"time", "level", "attr1", "device", "num"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,l1,d,d2,3,", + "1971-01-01T00:00:00.000Z,l1,d,d2,6,", + "1971-01-01T00:01:40.000Z,l1,d,d2,11,", + "1970-01-01T00:00:00.000Z,l1,c,d1,3,", + "1971-01-01T00:00:00.000Z,l1,c,d1,6,", + "1971-01-01T00:01:40.000Z,l1,c,d1,11,", + "1970-01-01T00:00:00.020Z,l2,yy,d1,2,", + "1971-01-01T00:00:00.100Z,l2,yy,d1,10,", + "1971-04-26T17:46:40.000Z,l2,yy,d1,12,", + "1970-01-01T00:00:00.020Z,l2,vv,d2,2,", + "1971-01-01T00:00:00.100Z,l2,vv,d2,10,", + "1971-04-26T17:46:40.000Z,l2,vv,d2,12,", + "1970-01-01T00:00:00.040Z,l3,null,d2,1,", + "1971-01-01T00:00:00.500Z,l3,null,d2,4,", + "1971-04-26T17:46:40.020Z,l3,null,d2,14,", + "1970-01-01T00:00:00.040Z,l3,t,d1,1,", + "1971-01-01T00:00:00.500Z,l3,t,d1,4,", + "1971-04-26T17:46:40.020Z,l3,t,d1,14,", + "1970-01-01T00:00:00.080Z,l4,null,d2,9,", + "1971-01-01T00:00:01.000Z,l4,null,d2,5,", + "1971-04-26T18:01:40.000Z,l4,null,d2,13,", + "1970-01-01T00:00:00.080Z,l4,null,d1,9,", + "1971-01-01T00:00:01.000Z,l4,null,d1,5,", + "1971-04-26T18:01:40.000Z,l4,null,d1,13,", + "1970-01-01T00:00:00.100Z,l5,null,d2,8,", + "1971-01-01T00:00:10.000Z,l5,null,d2,7,", + "1971-08-20T11:33:20.000Z,l5,null,d2,15,", + "1970-01-01T00:00:00.100Z,l5,null,d1,8,", + "1971-01-01T00:00:10.000Z,l5,null,d1,7,", + "1971-08-20T11:33:20.000Z,l5,null,d1,15,", + }; + tableResultSetEqualTest( + "select time,level,attr1,device,num from table0 order by level asc, attr1 desc nulls first, device desc, time asc", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select time,level,attr1,device,num from table0 as t order by level asc, t.attr1 desc nulls first, t.device desc, time asc", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sortTAGWithExpressionTest() { + String[] expectedHeader = new String[] {"level", "sum", "attr1", "device", "time"}; + String[] retArray = + new String[] { + "l1,65,d,d2,1971-01-01T00:01:40.000Z,", + "l1,65,c,d1,1971-01-01T00:01:40.000Z,", + "l1,234,d,d2,1970-01-01T00:00:00.000Z,", + "l1,234,c,d1,1970-01-01T00:00:00.000Z,", + "l1,1237,d,d2,1971-01-01T00:00:00.000Z,", + "l1,1237,c,d1,1971-01-01T00:00:00.000Z,", + "l2,57,yy,d1,1971-04-26T17:46:40.000Z,", + "l2,57,vv,d2,1971-04-26T17:46:40.000Z,", + "l2,242,yy,d1,1971-01-01T00:00:00.100Z,", + "l2,242,vv,d2,1971-01-01T00:00:00.100Z,", + "l2,436,yy,d1,1970-01-01T00:00:00.020Z,", + "l2,436,vv,d2,1970-01-01T00:00:00.020Z,", + "l3,13,t,d1,1970-01-01T00:00:00.040Z,", + "l3,13,null,d2,1970-01-01T00:00:00.040Z,", + "l3,217,t,d1,1971-01-01T00:00:00.500Z,", + "l3,217,null,d2,1971-01-01T00:00:00.500Z,", + "l3,245,t,d1,1971-04-26T17:46:40.020Z,", + "l3,245,null,d2,1971-04-26T17:46:40.020Z,", + "l4,52,null,d2,1970-01-01T00:00:00.080Z,", + "l4,52,null,d1,1970-01-01T00:00:00.080Z,", + "l4,61,null,d2,1971-01-01T00:00:01.000Z,", + "l4,61,null,d1,1971-01-01T00:00:01.000Z,", + "l4,67,null,d2,1971-04-26T18:01:40.000Z,", + "l4,67,null,d1,1971-04-26T18:01:40.000Z,", + "l5,220,null,d1,1971-01-01T00:00:10.000Z,", + "l5,220,null,d2,1971-01-01T00:00:10.000Z,", + "l5,250,null,d1,1971-08-20T11:33:20.000Z,", + "l5,250,null,d2,1971-08-20T11:33:20.000Z,", + "l5,4662,null,d1,1970-01-01T00:00:00.100Z,", + "l5,4662,null,d2,1970-01-01T00:00:00.100Z,", + }; + tableResultSetEqualTest( + "select level,cast(num+floatNum as int32) as sum,attr1,device,time from table0 order by level asc, cast(num+floatNum as int32) asc, attr1 desc", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void dateTypeTopKTest() { + String[] expectedHeader = new String[] {"time", "level", "attr1", "device", "num", "date"}; + String[] retArray = + new String[] { + "1971-08-20T11:33:20.000Z,l5,null,d2,15,2023-01-01,", + "1971-08-20T11:33:20.000Z,l5,null,d1,15,2022-01-01,", + "1971-04-26T18:01:40.000Z,l4,null,d2,13,null,", + "1971-04-26T18:01:40.000Z,l4,null,d1,13,null,", + "1971-04-26T17:46:40.020Z,l3,null,d2,14,null,", + }; + tableResultSetEqualTest( + "select time,level,attr1,device,num,date from table0 order by time desc,device desc limit 5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void projectSortTest() { + String[] expectedHeader = new String[] {"level", "attr1", "device", "num", "date"}; + String[] retArray = + new String[] { + "l2,yy,d1,2,null,", + "l2,yy,d1,10,null,", + "l2,yy,d1,12,null,", + "l1,c,d1,3,null,", + "l1,c,d1,6,null,", + "l1,c,d1,11,null,", + }; + tableResultSetEqualTest( + "select level,attr1,device,num,date from table0 order by attr2 desc,time limit 6", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "level", "attr2", "str"}; + retArray = + new String[] { + "1970-01-01T00:00:00.040Z,l3,a,apricot,", + "1970-01-01T00:00:00.040Z,l3,null,apricot,", + "1970-01-01T00:00:00.020Z,l2,null,pineapple,", + "1970-01-01T00:00:00.020Z,l2,zz,pineapple,", + "1970-01-01T00:00:00.000Z,l1,d,coconut,", + "1970-01-01T00:00:00.000Z,l1,c,coconut,", + }; + tableResultSetEqualTest( + "select time,level,attr2,str from table0 order by num+1,attr1 limit 6", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void coalesceTest() { + String[] expectedHeader = new String[] {"time", "level", "coalesce_attr2", "str"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.040Z,l3,a,apricot,", + "1970-01-01T00:00:00.040Z,l3,CCC,apricot,", + "1970-01-01T00:00:00.020Z,l2,CCC,pineapple,", + "1970-01-01T00:00:00.020Z,l2,zz,pineapple,", + "1970-01-01T00:00:00.000Z,l1,d,coconut,", + "1970-01-01T00:00:00.000Z,l1,c,coconut,", + }; + tableResultSetEqualTest( + "select time,level,coalesce(attr2, 'CCC', 'DDD') as coalesce_attr2,str from table0 order by num+1,attr1 limit 6", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // ========== SubQuery Test ========= + @Test + public void subQueryTest1() { + String[] expectedHeader = new String[] {"time", "level", "device", "add_num"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.100Z,l5,d1,9,", + "1971-01-01T00:00:01.000Z,l4,d1,6,", + "1971-01-01T00:00:10.000Z,l5,d1,8,", + "1971-04-26T18:01:40.000Z,l4,d1,14,", + "1971-08-20T11:33:20.000Z,l5,d1,16,", + "1970-01-01T00:00:00.080Z,l4,d2,10,", + }; + + tableResultSetEqualTest( + "SELECT time, level, device, add_num FROM (\n" + + "SELECT time, level, device, substring(str, 2) as cast_str, attr2, bignum, num+1 as add_num FROM table0 WHERE num>1 ORDER BY level DESC, time, device LIMIT 12\n" + + ") ORDER BY DEVICE,time OFFSET 1 LIMIT 6", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // ========================================================================= + // ============================ Aggregation Test =========================== + // ========================================================================= + @Test + public void aggregationTest() { + String[] expectedHeader = + new String[] { + "count_num", + "count_star", + "avg_num", + "count_num", + "count_attr2", + "avg_num", + "count_device", + "count_attr1", + "count_device", + "avg_floatnum", + "count_date", + "count_time", + "count_star" + }; + String[] retArray = + new String[] { + "30,30,8.0,30,12,8.0,30,15,30,529.0,2,30,30,", + }; + tableResultSetEqualTest( + "select count(num) as count_num, count(*) as count_star, avg(num) as avg_num, count(num) as count_num,\n" + + "count(attr2) as count_attr2, avg(num) as avg_num, count(device) as count_device,\n" + + "count(attr1) as count_attr1, count(device) as count_device, \n" + + "round(avg(floatnum)) as avg_floatnum, count(date) as count_date, " + + "count(time) as count_time, count(*) as count_star " + + "from table0", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count(num) as count_num, count(1) as count_star, avg(num) as avg_num, count(num) as count_num,\n" + + "count(attr2) as count_attr2, avg(num) as avg_num, count(device) as count_device,\n" + + "count(attr1) as count_attr1, count(device) as count_device, \n" + + "round(avg(floatnum)) as avg_floatnum, count(date) as count_date, " + + "count(time) as count_time, count(1) as count_star " + + "from table0", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "8,8,5.5,8,3,5.5,8,4,8,1341.0,0,8,8,", + }; + tableResultSetEqualTest( + "select count(num) as count_num, count(*) as count_star, avg(num) as avg_num, count(num) as count_num,\n" + + "count(attr2) as count_attr2, avg(num) as avg_num, count(device) as count_device,\n" + + "count(attr1) as count_attr1, count(device) as count_device, \n" + + "round(avg(floatnum)) as avg_floatnum, count(date) as count_date, " + + "count(time) as count_time, count(*) as count_star " + + "from table0 " + + "where time<200 and num>1 ", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count(num) as count_num, count(1) as count_star, avg(num) as avg_num, count(num) as count_num,\n" + + "count(attr2) as count_attr2, avg(num) as avg_num, count(device) as count_device,\n" + + "count(attr1) as count_attr1, count(device) as count_device, \n" + + "round(avg(floatnum)) as avg_floatnum, count(date) as count_date, " + + "count(time) as count_time, count(1) as count_star " + + "from table0 " + + "where time<200 and num>1 ", + expectedHeader, + retArray, + DATABASE_NAME); + + // TAG has null value + expectedHeader = + new String[] { + "count_num", + "avg_num", + "count_num", + "count_attr2", + "avg_num", + "count_device", + "count_attr1", + "count_device", + "avg_floatnum", + "count_date", + "count_level", + "count_time", + "count_star" + }; + retArray = + new String[] { + "8,5.125,8,4,5.125,7,4,7,1134.0,3,7,8,8,", + }; + tableResultSetEqualTest( + "select count(num) as count_num, avg(num) as avg_num, count(num) as count_num,\n" + + "count(attr2) as count_attr2, avg(num) as avg_num, count(device) as count_device,\n" + + "count(attr1) as count_attr1, count(device) as count_device, \n" + + "round(avg(floatnum)) as avg_floatnum, count(date) as count_date, count(level) as count_level," + + "count(time) as count_time, count(*) as count_star " + + "from table1 where device != 'd11' or device is null", + expectedHeader, + retArray, + DATABASE_NAME); + + // count star with subquery + // TODO(beyyes) add first/last blob type test + } + + @Test + public void globalAggregationTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = new String[] {"30,"}; + + sql = "SELECT count(num+1) from table0"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void countStarTest() { + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"1,1,"}; + sql = "select count(*),count(t1) from (select avg(num+1) as t1 from table0)"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = "select count(1),count(t1) from (select avg(num+1) as t1 from table0)"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"count_star"}; + retArray = + new String[] { + "30,", + }; + tableResultSetEqualTest( + "select count(*) as count_star from table0", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "select count(1) as count_star from table0", expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "1,", + }; + tableResultSetEqualTest( + "select count(*) as count_star from (select count(*) from table0)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count(1) as count_star from (select count(1) from table0)", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1,", + }; + tableResultSetEqualTest( + "select count(*) as count_star from (select count(*), avg(num) from table0)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count(1) as count_star from (select count(1), avg(num) from table0)", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"sum"}; + retArray = + new String[] { + "38.0,", + }; + tableResultSetEqualTest( + "select count_star + avg_num as sum from (select count(*) as count_star, avg(num) as avg_num from table0)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count_star + avg_num as sum from (select count(1) as count_star, avg(num) as avg_num from table0)", + expectedHeader, + retArray, + DATABASE_NAME); + + // TODO select count(*),count(t1) from (select avg(num+1) as t1 from table0) where time < 0 + + expectedHeader = buildHeaders(1); + retArray = + new String[] { + "10,", + }; + tableResultSetEqualTest( + "select count(*) from (select device from table0 group by device, level)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count(1) from (select device from table0 group by device, level)", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void groupByAggregationTest() { + expectedHeader = + new String[] { + "device", + "level", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num" + }; + retArray = + new String[] { + "d1,l1,3,3,3,0,3,3,3,20.0,", + "d1,l2,3,3,3,0,3,3,3,24.0,", + "d1,l3,3,3,3,0,3,3,3,19.0,", + "d1,l4,3,3,3,0,0,0,3,27.0,", + "d1,l5,3,3,3,1,0,0,3,30.0,", + "d2,l1,3,3,3,0,3,3,3,20.0,", + "d2,l2,3,3,3,0,3,0,3,24.0,", + "d2,l3,3,3,3,0,0,0,3,19.0,", + "d2,l4,3,3,3,0,0,0,3,27.0,", + "d2,l5,3,3,3,1,0,0,3,30.0,", + }; + sql = + "select device, level, " + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num " + + "from table0 group by device,level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, " + + "count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num " + + "from table0 group by device,level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num" + }; + retArray = + new String[] { + "d1,3,3,3,0,3,3,3,20.0,", + }; + sql = + "select device, " + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num " + + "from table0 where device='d1' and level='l1' group by device order by device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, " + + "count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num " + + "from table0 where device='d1' and level='l1' group by device order by device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"device", "level"}; + retArray = + new String[] { + "d1,l1,", "d1,l2,", "d1,l3,", "d1,l4,", "d1,l5,", "d2,l1,", "d2,l2,", "d2,l3,", "d2,l4,", + "d2,l5,", + }; + sql = "select device,level from table0 group by device,level order by device,level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"device", "level", "attr1", "bin"}; + retArray = + new String[] { + "d1,l1,c,1970-01-01T00:00:00.000Z,", + "d1,l1,c,1971-01-01T00:00:00.000Z,", + "d1,l2,yy,1970-01-01T00:00:00.000Z,", + "d1,l2,yy,1971-01-01T00:00:00.000Z,", + "d1,l2,yy,1971-04-26T00:00:00.000Z,", + "d1,l3,t,1970-01-01T00:00:00.000Z,", + "d1,l3,t,1971-01-01T00:00:00.000Z,", + "d1,l3,t,1971-04-26T00:00:00.000Z,", + "d1,l4,null,1970-01-01T00:00:00.000Z,", + "d1,l4,null,1971-01-01T00:00:00.000Z,", + "d1,l4,null,1971-04-26T00:00:00.000Z,", + "d1,l5,null,1970-01-01T00:00:00.000Z,", + "d1,l5,null,1971-01-01T00:00:00.000Z,", + "d1,l5,null,1971-08-20T00:00:00.000Z,", + "d2,l1,d,1970-01-01T00:00:00.000Z,", + "d2,l1,d,1971-01-01T00:00:00.000Z,", + "d2,l2,vv,1970-01-01T00:00:00.000Z,", + "d2,l2,vv,1971-01-01T00:00:00.000Z,", + "d2,l2,vv,1971-04-26T00:00:00.000Z,", + "d2,l3,null,1970-01-01T00:00:00.000Z,", + "d2,l3,null,1971-01-01T00:00:00.000Z,", + "d2,l3,null,1971-04-26T00:00:00.000Z,", + "d2,l4,null,1970-01-01T00:00:00.000Z,", + "d2,l4,null,1971-01-01T00:00:00.000Z,", + "d2,l4,null,1971-04-26T00:00:00.000Z,", + "d2,l5,null,1970-01-01T00:00:00.000Z,", + "d2,l5,null,1971-01-01T00:00:00.000Z,", + "d2,l5,null,1971-08-20T00:00:00.000Z,", + }; + sql = + "select device,level,attr1,date_bin(1d,time) as bin from table0 group by 4,device,level,attr1 order by device,level,attr1,bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"attr1", "attr2"}; + retArray = + new String[] { + "c,d,", "d,c,", "t,a,", "vv,null,", "yy,zz,", "null,null,", + }; + sql = "select attr1,attr2 from table0 group by attr1,attr2 order by attr1,attr2"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void groupByDateBinTest() { + expectedHeader = + new String[] { + "device", + "level", + "bin", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "avg_num" + }; + retArray = + new String[] { + "d1,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,", + "d1,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,", + "d1,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,", + "d1,l2,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,11.0,", + "d1,l3,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,1.0,", + "d1,l3,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,9.0,", + "d1,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,", + "d1,l4,1971-01-01T00:00:00.000Z,2,2,2,0,0,0,2,9.0,", + "d1,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,", + "d1,l5,1971-01-01T00:00:00.000Z,2,2,2,1,0,0,2,11.0,", + "d2,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,", + "d2,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,", + "d2,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,0,1,2.0,", + "d2,l2,1971-01-01T00:00:00.000Z,2,2,2,0,2,0,2,11.0,", + "d2,l3,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,1.0,", + "d2,l3,1971-01-01T00:00:00.000Z,2,2,2,0,0,0,2,9.0,", + "d2,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,", + "d2,l4,1971-01-01T00:00:00.000Z,2,2,2,0,0,0,2,9.0,", + "d2,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,", + "d2,l5,1971-01-01T00:00:00.000Z,2,2,2,1,0,0,2,11.0,", + }; + sql = + "select device, level, date_bin(1y, time) as bin," + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 group by 3, device, level order by device, level, bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, date_bin(1y, time) as bin," + + "count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 group by 3, device, level order by device, level, bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "d1,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,", + "d1,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,", + "d1,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,", + "d1,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,10.0,", + "d1,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,1,1,12.0,", + "d1,l3,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,1.0,", + "d1,l3,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,4.0,", + "d1,l3,1971-04-26T00:00:00.000Z,1,1,1,0,1,1,1,14.0,", + "d1,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,", + "d1,l4,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,5.0,", + "d1,l4,1971-04-26T00:00:00.000Z,1,1,1,0,0,0,1,13.0,", + "d1,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,", + "d1,l5,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,7.0,", + "d1,l5,1971-08-20T00:00:00.000Z,1,1,1,1,0,0,1,15.0,", + "d2,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,", + "d2,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,", + "d2,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,0,1,2.0,", + "d2,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,0,1,10.0,", + "d2,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,0,1,12.0,", + "d2,l3,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,1.0,", + "d2,l3,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,4.0,", + "d2,l3,1971-04-26T00:00:00.000Z,1,1,1,0,0,0,1,14.0,", + "d2,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,", + "d2,l4,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,5.0,", + "d2,l4,1971-04-26T00:00:00.000Z,1,1,1,0,0,0,1,13.0,", + "d2,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,", + "d2,l5,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,7.0,", + "d2,l5,1971-08-20T00:00:00.000Z,1,1,1,1,0,0,1,15.0,", + }; + sql = + "select device, level, date_bin(1d, time) as bin," + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 group by 3, device, level order by device, level, bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, date_bin(1d, time) as bin," + + "count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 group by 3, device, level order by device, level, bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "d1,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,", + "d1,l1,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,6.0,", + "d1,l1,1971-01-01T00:01:40.000Z,1,1,1,0,1,1,1,11.0,", + "d1,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,", + "d1,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,10.0,", + "d1,l2,1971-04-26T17:46:40.000Z,1,1,1,0,1,1,1,12.0,", + "d1,l3,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,1.0,", + "d1,l3,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,4.0,", + "d1,l3,1971-04-26T17:46:40.000Z,1,1,1,0,1,1,1,14.0,", + "d1,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,", + "d1,l4,1971-01-01T00:00:01.000Z,1,1,1,0,0,0,1,5.0,", + "d1,l4,1971-04-26T18:01:40.000Z,1,1,1,0,0,0,1,13.0,", + "d1,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,", + "d1,l5,1971-01-01T00:00:10.000Z,1,1,1,0,0,0,1,7.0,", + "d1,l5,1971-08-20T11:33:20.000Z,1,1,1,1,0,0,1,15.0,", + "d2,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,", + "d2,l1,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,6.0,", + "d2,l1,1971-01-01T00:01:40.000Z,1,1,1,0,1,1,1,11.0,", + "d2,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,0,1,2.0,", + "d2,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,0,1,10.0,", + "d2,l2,1971-04-26T17:46:40.000Z,1,1,1,0,1,0,1,12.0,", + "d2,l3,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,1.0,", + "d2,l3,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,4.0,", + "d2,l3,1971-04-26T17:46:40.000Z,1,1,1,0,0,0,1,14.0,", + "d2,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,", + "d2,l4,1971-01-01T00:00:01.000Z,1,1,1,0,0,0,1,5.0,", + "d2,l4,1971-04-26T18:01:40.000Z,1,1,1,0,0,0,1,13.0,", + "d2,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,", + "d2,l5,1971-01-01T00:00:10.000Z,1,1,1,0,0,0,1,7.0,", + "d2,l5,1971-08-20T11:33:20.000Z,1,1,1,1,0,0,1,15.0,", + }; + sql = + "select device, level, date_bin(1s, time) as bin," + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 group by 3, device, level order by device, level, bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, date_bin(1s, time) as bin," + + "count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 group by 3, device, level order by device, level, bin"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // only group by date_bin + expectedHeader = new String[] {"bin"}; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,", "1971-01-01T00:00:00.000Z,", "1971-04-26T00:00:00.000Z," + }; + sql = + "select date_bin(1d, time) as bin from table0 where device='d1' and level='l2' group by 1"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "bin", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "avg_num" + }; + sql = + "select date_bin(1s, time) as bin," + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 where device='d1' and level='l2' group by 1"; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,", + "1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,10.0,", + "1971-04-26T17:46:40.000Z,1,1,1,0,1,1,1,12.0," + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select date_bin(1s, time) as bin," + + "count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num " + + "from table0 where device='d1' and level='l2' group by 1"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // flush multi times, generated multi tsfile + expectedHeader = buildHeaders(1); + sql = "select date_bin(40ms,time), first(time) from table1 where device='d11' group by 1"; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.000Z,", + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.000Z," + }; + + // TODO(beyyes) test below + // sql = "select count(*) from (\n" + + // "\tselect device, level, date_bin(1d, time) as bin, \n" + + // "\tcount(num) as count_num, count(*) as count_star, count(device) as + // count_device, + // count(date) as count_date, count(attr1) as count_attr1, count(attr2) as count_attr2, + // count(time) as count_time, avg(num) as avg_num \n" + + // "\tfrom table0 \n" + + // "\tgroup by 3, device, level order by device, level, bin\n" + + // ")\n"; + } + + @Test + public void aggregationNoDataTest() { + expectedHeader = + new String[] { + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num", + "avg_num" + }; + + retArray = + new String[] { + "0,0,0,0,0,0,0,null,null,", + }; + sql = + "select count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2,2,2,0,2,1,2,24.0,12.0,", + }; + sql = + "select count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", + "level", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num", + "avg_num" + }; + retArray = new String[] {}; + sql = + "select device, level, count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 group by device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 group by device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + retArray = new String[] {"d1,l2,1,1,1,0,1,1,1,12.0,12.0,", "d2,l2,1,1,1,0,1,0,1,12.0,12.0,"}; + sql = + "select device, level, count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00 group by device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, count(num) as count_num, count(1) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00 group by device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", + "level", + "bin", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num", + "avg_num" + }; + retArray = new String[] {}; + sql = + "select device, level, date_bin(1d, time) as bin, count(num) as count_num, count(*) as count_star, " + + "count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 group by 3, device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, date_bin(1d, time) as bin, count(num) as count_num, count(1) as count_star, " + + "count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 group by 3, device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + retArray = + new String[] { + "d1,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,1,1,12.0,12.0,", + "d2,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,0,1,12.0,12.0," + }; + sql = + "select device, level, date_bin(1d, time) as bin, count(num) as count_num, count(*) as count_star, " + + "count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00 group by 3, device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select device, level, date_bin(1d, time) as bin, count(num) as count_num, count(1) as count_star, " + + "count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00 group by 3, device, level order by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // queried device is not exist + expectedHeader = buildHeaders(3); + sql = "select count(*), count(num), sum(num) from table0 where device='d_not_exist'"; + retArray = new String[] {"0,0,null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = "select count(1), count(num), sum(num) from table0 where device='d_not_exist'"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select count(*), count(num), sum(num) from table0 where device='d_not_exist1' or device='d_not_exist2'"; + retArray = new String[] {"0,0,null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + sql = + "select count(1), count(num), sum(num) from table0 where device='d_not_exist1' or device='d_not_exist2'"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // no data in given time range (push-down) + sql = "select count(*), count(num), sum(num) from table0 where time>2100-04-26T18:01:40.000"; + retArray = new String[] {"0,0,null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // no data in given time range (no push-down) + sql = "select count(*), count(num+1), sum(num) from table0 where time>2100-04-26T18:01:40.000"; + retArray = new String[] {"0,0,null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // only one device has data in queried time + expectedHeader = buildHeaders(2); + sql = "select count(num),sum(num) from table1 where time=0"; + retArray = new String[] {"2,6.0,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void lastFirstMaxMinTest() { + expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13", + }; + retArray = + new String[] { + "1971-04-26T18:01:40.000Z,d1,l4,null,null,13,2107483648,54.12,lychee,true,null,0x108dcd62,2024-09-15T06:15:35.000Z,test-string1,", + }; + sql = + "select last(time),last(device),last(level),last(attr1),last(attr2),last(num),last(bignum),last(floatnum),last(str),last(bool),last(date),last(blob),last(ts),last(stringv) from table0 where device='d1' and level='l4'"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + retArray = + new String[] { + "1970-01-01T00:00:00.080Z,d1,l4,null,null,9,2147483646,43.12,apple,false,null,0x108dcd62,2024-09-15T06:15:35.000Z,test-string1,", + }; + sql = + "select first(time),first(device),first(level),first(attr1),first(attr2),first(num),first(bignum),first(floatnum),first(str),first(bool),first(date),first(blob),first(ts),first(stringv) from table0 where device='d1' and level='l4'"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(30); + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,d2,l5,yy,zz,15,3147483648,4654.231,watermelon,true,2023-01-01,0x108dcd63,2024-09-25T06:15:35.000Z,test-string3,6666.8,1970-01-01T00:00:00.000Z,d1,l1,c,a,1,2107483648,12.123,apple,false,2022-01-01,0x108dcd62,2024-08-01T06:15:35.000Z,test-string1,6666.3,", + }; + sql = + "select max(time),max(device),max(level),max(attr1),max(attr2),max(num),max(bignum),max(floatnum),max(str),max(bool),max(date),max(blob),max(ts),max(stringv),max(doubleNum),min(time),min(device),min(level),min(attr1),min(attr2),min(num),min(bignum),min(floatnum),min(str),min(bool),min(date),min(blob),min(ts),min(stringv),min(doubleNum) from table0"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(24); + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,d2,l5,yy,zz,15,3147483648,4654.231,2023-01-01,1971-01-01T00:01:40.000Z,test-string3,6666.8,1970-01-01T00:00:00.000Z,d1,l1,c,a,1,2107483648,12.123,2022-01-01,1970-01-01T00:00:00.020Z,test-string1,6666.3,", + }; + sql = + "select max(time),max(device),max(level),max(attr1),max(attr2),max(num),max(bignum),max(floatnum),max(date),max(ts),max(stringv),max(doubleNum),min(time),min(device),min(level),min(attr1),min(attr2),min(num),min(bignum),min(floatnum),min(date),min(ts),min(stringv),min(doubleNum) from table0"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", "level", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", + }; + retArray = + new String[] { + "d1,l1,11,2947483648,1231.21,3,2147468648,54.121,", + "d1,l2,12,3147483648,434.12,2,2146483648,45.231,", + "d1,l3,14,2907483648,231.34,1,2147493648,12.123,", + "d1,l4,13,2149783648,56.32,5,2107483648,43.12,", + "d1,l5,15,3147483648,4654.231,7,2147483964,213.112,", + "d2,l1,11,2947483648,1231.21,3,2147468648,54.121,", + "d2,l2,12,3147483648,434.12,2,2146483648,45.231,", + "d2,l3,14,2907483648,231.34,1,2147493648,12.123,", + "d2,l4,13,2149783648,56.32,5,2107483648,43.12,", + "d2,l5,15,3147483648,4654.231,7,2147483964,213.112,", + }; + sql = + "select device,level,max(num),max(bignum),max(floatnum),min(num),min(bignum),min(floatnum) from table0 group by device,level order by device,level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // no push-down, test GroupedAccumulator + expectedHeader = + new String[] { + "device", "level", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12" + }; + retArray = + new String[] { + "d1,l1,11,2947483648,1231.21,3,2147468648,54.121,pitaya,banana,true,false,3,", + "d1,l2,12,3147483648,434.12,2,2146483648,45.231,strawberry,pineapple,true,false,3,", + "d1,l3,14,2907483648,231.34,1,2147493648,12.123,peach,apricot,true,false,3,", + "d1,l4,13,2149783648,56.32,5,2107483648,43.12,orange,apple,true,false,3,", + "d1,l5,15,3147483648,4654.231,7,2147483964,213.112,watermelon,lemon,true,true,3,", + "d2,l1,11,2947483648,1231.21,3,2147468648,54.121,pitaya,banana,true,false,3,", + "d2,l2,12,3147483648,434.12,2,2146483648,45.231,strawberry,pineapple,true,false,3,", + "d2,l3,14,2907483648,231.34,1,2147493648,12.123,peach,apricot,true,false,3,", + "d2,l4,13,2149783648,56.32,5,2107483648,43.12,orange,apple,true,false,3,", + "d2,l5,15,3147483648,4654.231,7,2147483964,213.112,watermelon,lemon,true,true,3,", + }; + sql = + "select device,level,max(num),max(bignum),max(floatnum),min(num),min(bignum),min(floatnum),max(str),min(str),max(bool),min(bool),count(num+1) from table0 group by device,level order by device,level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void lastByFirstByTest() { + String[] expectedHeader1 = buildHeaders(13); + String[] expectedHeader2 = buildHeaders(14); + + sql = + "select last_by(time,time),last_by(device,time),last_by(level,time),last_by(attr1,time),last_by(attr2,time),last_by(num,time),last_by(bignum,time),last_by(floatnum,time),last_by(str,time),last_by(bool,time),last_by(date,time),last_by(ts,time),last_by(stringv,time) from table0 where device='d2'"; + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,d2,l5,null,null,15,3147483648,235.213,watermelon,true,2023-01-01,null,null,", + }; + repeatTest(sql, expectedHeader1, retArray, DATABASE_NAME, 2); + + sql = + "select last_by(time,time),last_by(device,time),last_by(level,time),last_by(attr1,time),last_by(attr2,time),last_by(num,time),last_by(bignum,time),last_by(floatnum,time),last_by(str,time),last_by(bool,time),last_by(date,time),last_by(ts,time),last_by(stringv,time),last_by(blob,time) from table0 where device='d2'"; + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,d2,l5,null,null,15,3147483648,235.213,watermelon,true,2023-01-01,null,null,null,", + }; + repeatTest(sql, expectedHeader2, retArray, DATABASE_NAME, 2); + + sql = + "select last_by(time,time),last_by(time,device),last_by(time,level),last_by(time,attr1),last_by(time,attr2),last_by(time,num),last_by(time,bignum),last_by(time,floatnum),last_by(time,str),last_by(time,bool),last_by(time,date),last_by(time,ts),last_by(time,stringv) from table0 where device='d2'"; + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-04-26T17:46:40.000Z,1971-01-01T00:01:40.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-01-01T00:01:40.000Z,1971-01-01T00:01:40.000Z,", + }; + repeatTest(sql, expectedHeader1, retArray, DATABASE_NAME, 2); + + sql = + "select last_by(time,time),last_by(time,device),last_by(time,level),last_by(time,attr1),last_by(time,attr2),last_by(time,num),last_by(time,bignum),last_by(time,floatnum),last_by(time,str),last_by(time,bool),last_by(time,date),last_by(time,ts),last_by(time,stringv),last_by(time,blob) from table0 where device='d2'"; + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-04-26T17:46:40.000Z,1971-01-01T00:01:40.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-01-01T00:01:40.000Z,1971-01-01T00:01:40.000Z,1970-01-01T00:00:00.080Z,", + }; + repeatTest(sql, expectedHeader2, retArray, DATABASE_NAME, 2); + + String[] expectedHeader11 = buildHeaders(expectedHeader1.length * 2); + sql = + "select last_by(time,time),last_by(device,time),last_by(level,time),last_by(attr1,time),last_by(attr2,time),last_by(num,time),last_by(bignum,time),last_by(floatnum,time),last_by(str,time),last_by(bool,time),last_by(date,time),last_by(ts,time),last_by(stringv,time),last_by(time,time),last_by(time,device),last_by(time,level),last_by(time,attr1),last_by(time,attr2),last_by(time,num),last_by(time,bignum),last_by(time,floatnum),last_by(time,str),last_by(time,bool),last_by(time,date),last_by(time,ts),last_by(time,stringv) from table0 where device='d2'"; + retArray = + new String[] { + "1971-08-20T11:33:20.000Z,d2,l5,null,null,15,3147483648,235.213,watermelon,true,2023-01-01,null,null,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-04-26T17:46:40.000Z,1971-01-01T00:01:40.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1971-01-01T00:01:40.000Z,1971-01-01T00:01:40.000Z,", + }; + repeatTest(sql, expectedHeader11, retArray, DATABASE_NAME, 2); + + sql = + "select first_by(time,time),first_by(device,time),first_by(level,time),first_by(attr1,time),first_by(attr2,time),first_by(num,time),first_by(bignum,time),first_by(floatnum,time),first_by(str,time),first_by(bool,time),first_by(date,time),first_by(ts,time),first_by(stringv,time) from table0 where device='d2' and time>80"; + retArray = + new String[] { + "1970-01-01T00:00:00.100Z,d2,l5,null,null,8,2147483964,4654.231,papaya,true,null,null,null,", + }; + tableResultSetEqualTest(sql, expectedHeader1, retArray, DATABASE_NAME); + + sql = + "select first_by(time,time),first_by(time,device),first_by(time,level),first_by(time,attr1),first_by(time,attr2),first_by(time,num),first_by(time,bignum),first_by(time,floatnum),first_by(time,str),first_by(time,bool),first_by(time,date),first_by(time,ts),first_by(time,stringv) from table0 where device='d2' and time>80"; + retArray = + new String[] { + "1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.100Z,1971-01-01T00:00:00.000Z,1971-01-01T00:00:00.000Z,1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.100Z,1971-08-20T11:33:20.000Z,1971-01-01T00:01:40.000Z,1971-01-01T00:01:40.000Z,", + }; + tableResultSetEqualTest(sql, expectedHeader1, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(3); + sql = "select last_by(time,num+1),last_by(num+1,time),last_by(num+1,floatnum+1) from table0"; + retArray = new String[] {"1971-08-20T11:33:20.000Z,16,16,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxByMinByExtremeTest() { + expectedHeader = buildHeaders(10); + sql = + "select max_by(time,floatnum),min_by(time,floatnum),max_by(time,date),min_by(time,date),max_by(time,floatnum),min_by(time,floatnum),max_by(time,ts),min_by(time,ts),max_by(time,stringv),min_by(time,stringv) from table0"; + retArray = + new String[] { + "1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.040Z,1971-08-20T11:33:20.000Z,1971-08-20T11:33:20.000Z,1970-01-01T00:00:00.100Z,1970-01-01T00:00:00.040Z,1971-01-01T00:00:10.000Z,1971-01-01T00:01:40.000Z,1971-01-01T00:01:40.000Z,1971-01-01T00:00:01.000Z,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(30); + sql = + "select max_by(time,blob),max_by(device,blob),max_by(level,blob),max_by(attr1,blob),max_by(attr2,blob),max_by(num,blob),max_by(bignum,blob),max_by(floatnum,blob),max_by(str,blob),max_by(bool,blob),max_by(date,blob),max_by(blob,blob),max_by(ts,blob),max_by(stringv,blob),max_by(doubleNum,blob),min_by(time,blob),min_by(device,blob),min_by(level,blob),min_by(attr1,blob),min_by(attr2,blob),min_by(num,blob),min_by(bignum,blob),min_by(floatnum,blob),min_by(str,blob),min_by(bool,blob),min_by(date,blob),min_by(blob,blob),min_by(ts,blob),min_by(stringv,blob),min_by(doubleNum,blob) from table0"; + retArray = + new String[] { + "1971-01-01T00:00:10.000Z,d1,l5,null,null,7,2147983648,213.112,lemon,true,null,0x108dcd63,2024-09-25T06:15:35.000Z,null,null,1970-01-01T00:00:00.020Z,d1,l2,yy,zz,2,2147483648,434.12,pineapple,true,null,0x108dcd62,2024-09-24T06:15:35.000Z,null,6666.8,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(3); + sql = "select extreme(num),extreme(bignum),extreme(floatnum) from table0"; + retArray = new String[] {"15,3147483648,4654.231,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // no push-down, test GroupedAccumulator + expectedHeader = buildHeaders(5); + retArray = + new String[] { + "1971-01-01T00:01:40.000Z,1971-01-01T00:00:00.000Z,1971-01-01T00:00:00.000Z,1971-01-01T00:01:40.000Z,3,", + "1971-04-26T17:46:40.000Z,1970-01-01T00:00:00.020Z,1970-01-01T00:00:00.020Z,1971-04-26T17:46:40.000Z,3,", + "1971-01-01T00:00:00.500Z,1970-01-01T00:00:00.040Z,1970-01-01T00:00:00.040Z,1971-04-26T17:46:40.020Z,3,", + "1971-01-01T00:00:01.000Z,1970-01-01T00:00:00.080Z,1971-04-26T18:01:40.000Z,1971-01-01T00:00:01.000Z,3,", + "1971-08-20T11:33:20.000Z,1971-01-01T00:00:10.000Z,1971-08-20T11:33:20.000Z,1970-01-01T00:00:00.100Z,3,", + "1971-01-01T00:01:40.000Z,1971-01-01T00:00:00.000Z,1971-01-01T00:00:00.000Z,1971-01-01T00:01:40.000Z,3,", + "1971-04-26T17:46:40.000Z,1970-01-01T00:00:00.020Z,1970-01-01T00:00:00.020Z,1971-04-26T17:46:40.000Z,3,", + "1971-01-01T00:00:00.500Z,1970-01-01T00:00:00.040Z,1970-01-01T00:00:00.040Z,1971-04-26T17:46:40.020Z,3,", + "1971-01-01T00:00:01.000Z,1970-01-01T00:00:00.080Z,1971-04-26T18:01:40.000Z,1971-01-01T00:00:01.000Z,3,", + "1971-08-20T11:33:20.000Z,1971-01-01T00:00:10.000Z,1971-08-20T11:33:20.000Z,1970-01-01T00:00:00.100Z,3,", + }; + sql = + "select max_by(time,str),min_by(time,str),max_by(time,bool),min_by(time,bool),count(num+1) from table0 group by device,level order by device,level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void modeTest() { + expectedHeader = buildHeaders(15); + sql = + "select mode(time),mode(device),mode(level),mode(attr1),mode(attr2),mode(num),mode(bignum),mode(floatnum),mode(date),mode(str),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum) from table0 where device='d2' and level='l4' and time=80"; + retArray = + new String[] { + "1970-01-01T00:00:00.080Z,d2,l4,null,null,9,2147483646,43.12,null,apple,false,null,2024-09-20T06:15:35.000Z,test-string2,6666.7,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(10); + sql = + "select mode(device),mode(level),mode(attr1),mode(attr2),mode(date),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum) from table0 where device='d2' and level='l1'"; + retArray = + new String[] { + "d2,l1,d,c,null,false,null,null,null,null,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(1); + sql = + "select mode(stringv) from table0 where device='d2' and level='l1' and stringv is not null"; + retArray = + new String[] { + "test-string3,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // no push-down, test GroupedAccumulator + expectedHeader = buildHeaders(16); + sql = + "select mode(time),mode(device),mode(level),mode(attr1),mode(attr2),mode(num),mode(bignum),mode(floatnum),mode(date),mode(str),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum),count(num+1) from table0 where device='d2' and level='l4' and time=80 group by device, level"; + retArray = + new String[] { + "1970-01-01T00:00:00.080Z,d2,l4,null,null,9,2147483646,43.12,null,apple,false,null,2024-09-20T06:15:35.000Z,test-string2,6666.7,1,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(11); + sql = + "select mode(device),mode(level),mode(attr1),mode(attr2),mode(date),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum),count(num+1) from table0 where device='d2' and level='l1' group by device, level"; + retArray = + new String[] { + "d2,l1,d,c,null,false,null,null,null,null,3,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(2); + sql = + "select mode(stringv),count(num+1) from table0 where device='d2' and level='l1' and stringv is not null group by device, level"; + retArray = + new String[] { + "test-string3,1,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void varianceTest() { + expectedHeader = buildHeaders(18); + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1) from table0 where device='d2' and level='l4'"; + retArray = + new String[] { + "16.0,10.7,16.0,4.0,3.3,4.0,50.0,33.3,50.0,7.1,5.8,7.1,null,0.0,null,null,0.0,null,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(19); + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1), count(num+1) from table0 where device='d2' and level='l4' group by device, level"; + retArray = + new String[] { + "16.0,10.7,16.0,4.0,3.3,4.0,50.0,33.3,50.0,7.1,5.8,7.1,null,0.0,null,null,0.0,null,3,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1), count(num+1) from table0 group by device"; + retArray = + new String[] { + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.1,0.1,0.1,0.4,0.2,0.4,15,", + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.0,0.0,0.0,0.2,0.2,0.2,15," + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(18); + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1) from table0 group by device"; + retArray = + new String[] { + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.1,0.1,0.1,0.4,0.2,0.4,", + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.0,0.0,0.0,0.2,0.2,0.2," + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + // ================================================================== + // ============================ Join Test =========================== + // ================================================================== + // no filter + @Test + public void selfTimeColumnInnerJoinTest1() { + String[] expectedHeader = + new String[] {"time", "device", "level", "num", "device", "attr2", "num", "str"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d2,l1,3,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,d2,l1,3,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,d1,l2,2,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d1,l2,2,d2,null,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,l2,2,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,l2,2,d2,null,2,pineapple," + }; + + // join on + String sql = + "SELECT t1.time as time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 JOIN table0 t2 ON t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device OFFSET 2 LIMIT 6"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // implicit join + sql = + "SELECT t1.time as time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1, table0 t2 WHERE t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device OFFSET 2 LIMIT 6"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // join using + sql = + "SELECT time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 JOIN table0 t2 USING(time)\n" + + "ORDER BY time, t1.device, t2.device OFFSET 2 LIMIT 6"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + // has filter + @Test + public void selfTimeColumnInnerJoinTest2() { + String[] expectedHeader = + new String[] {"time", "device", "level", "t1_num_add", "device", "attr2", "num", "str"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.080Z,d1,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d2,null,9,apple,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d2,null,5,orange,", + }; + + // join on + String sql = + "SELECT t2.time,t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE TIME>=80 AND level!='l1' AND cast(num as double)>0) t1 \n" + + "JOIN (SELECT * FROM table0 WHERE TIME<=31536001000 AND floatNum<1000 AND device in ('d1','d2')) t2 \n" + + "ON t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT t2.time,t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0) t1 \n" + + "JOIN (SELECT * FROM table0) t2 ON t1.time = t2.time \n" + + "WHERE t1.TIME>=80 AND cast(t1.num as double)>0 AND t1.level!='l1' \n" + + "AND t2.time<=31536001000 AND t2.floatNum<1000 AND t2.device in ('d1','d2')\n" + + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT t2.time,t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE time>=80) t1 \n" + + "JOIN (SELECT * FROM table0 WHERE floatNum<1000) t2 ON t1.time = t2.time \n" + + "WHERE cast(t1.num as double)>0 AND t1.level!='l1' \n" + + "AND t2.time<=31536001000 AND t2.device in ('d1','d2')\n" + + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // implicit join + sql = + "SELECT t2.time,t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE TIME>=80 AND level!='l1' AND cast(num as double)>0) t1, \n" + + " (SELECT * FROM table0 WHERE TIME<=31536001000 AND floatNum<1000 AND device in ('d1','d2')) t2 \n" + + "WHERE t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT t2.time,t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0) t1, \n" + + " (SELECT * FROM table0) t2 \n" + + "WHERE t1.time=t2.time AND t1.TIME>=80 AND cast(t1.num as double)>0 AND t1.level!='l1' \n" + + "AND t2.time<=31536001000 AND t2.floatNum<1000 AND t2.device in ('d1','d2')\n" + + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT t2.time,t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE time>=80) t1, \n" + + " (SELECT * FROM table0 WHERE floatNum<1000) t2 \n" + + "WHERE t1.time=t2.time AND cast(t1.num as double)>0 AND t1.level!='l1' \n" + + "AND t2.time<=31536001000 AND t2.device in ('d1','d2')\n" + + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // join using + sql = + "SELECT time, t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE time>=80) t1 \n" + + "JOIN (SELECT * FROM table0 WHERE floatNum<1000) t2 USING(time) \n" + + "WHERE cast(t1.num as double)>0 AND t1.level!='l1' \n" + + "AND time<=31536001000 AND t2.device in ('d1','d2')\n" + + "ORDER BY time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + // no filter + @Test + public void timeColumnOuterJoinTest1() { + expectedHeader = + new String[] {"time", "device", "level", "num", "device", "attr2", "num", "str"}; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,l1,3,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,d1,l1,3,d2,c,3,coconut,", + "1970-01-01T00:00:00.000Z,d2,l1,3,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,d2,l1,3,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,d1,l2,2,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d1,l2,2,d2,null,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,l2,2,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,l2,2,d2,null,2,pineapple,", + "1970-01-01T00:00:00.040Z,d1,l3,1,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,d1,l3,1,d2,null,1,apricot,", + "1970-01-01T00:00:00.040Z,d2,l3,1,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,d2,l3,1,d2,null,1,apricot,", + "1970-01-01T00:00:00.080Z,d1,l4,9,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,9,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,9,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,9,d2,null,9,apple,", + "1970-01-01T00:00:00.100Z,d1,l5,8,d1,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d1,l5,8,d2,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d2,l5,8,d1,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d2,l5,8,d2,null,8,papaya,", + "1971-01-01T00:00:00.000Z,d1,l1,6,d1,d,6,banana,", + "1971-01-01T00:00:00.000Z,d1,l1,6,d2,c,6,banana,", + "1971-01-01T00:00:00.000Z,d2,l1,6,d1,d,6,banana,", + "1971-01-01T00:00:00.000Z,d2,l1,6,d2,c,6,banana,", + "1971-01-01T00:00:00.100Z,d1,l2,10,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,10,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,10,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,10,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,4,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,4,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,4,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,4,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,5,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,5,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,5,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,5,d2,null,5,orange,", + "1971-01-01T00:00:10.000Z,d1,l5,7,d1,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d1,l5,7,d2,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d2,l5,7,d1,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d2,l5,7,d2,null,7,lemon,", + "1971-01-01T00:01:40.000Z,d1,l1,11,d1,d,11,pitaya,", + "1971-01-01T00:01:40.000Z,d1,l1,11,d2,c,11,pitaya,", + "1971-01-01T00:01:40.000Z,d2,l1,11,d1,d,11,pitaya,", + "1971-01-01T00:01:40.000Z,d2,l1,11,d2,c,11,pitaya,", + "1971-04-26T17:46:40.000Z,d1,l2,12,d1,zz,12,strawberry,", + "1971-04-26T17:46:40.000Z,d1,l2,12,d2,null,12,strawberry,", + "1971-04-26T17:46:40.000Z,d2,l2,12,d1,zz,12,strawberry,", + "1971-04-26T17:46:40.000Z,d2,l2,12,d2,null,12,strawberry,", + "1971-04-26T17:46:40.020Z,d1,l3,14,d1,a,14,cherry,", + "1971-04-26T17:46:40.020Z,d1,l3,14,d2,null,14,cherry,", + "1971-04-26T17:46:40.020Z,d2,l3,14,d1,a,14,cherry,", + "1971-04-26T17:46:40.020Z,d2,l3,14,d2,null,14,cherry,", + "1971-04-26T18:01:40.000Z,d1,l4,13,d1,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d1,l4,13,d2,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d2,l4,13,d1,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d2,l4,13,d2,null,13,lychee,", + "1971-08-20T11:33:20.000Z,d1,l5,15,d1,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d1,l5,15,d2,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d2,l5,15,d1,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d2,l5,15,d2,null,15,watermelon,", + }; + + // join on + String sql = + "SELECT t1.time as time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 %s JOIN table0 t2 ON t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device"; + + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest(String.format(sql, INNER), expectedHeader, retArray, DATABASE_NAME); + + // join using + sql = + "SELECT time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 %s JOIN table0 t2 USING(time)\n" + + "ORDER BY time, t1.device, t2.device"; + + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest(String.format(sql, INNER), expectedHeader, retArray, DATABASE_NAME); + + sql = + "select t1.time as time1, t2.time as time2, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 from tableA t1 %s join tableB t2 on t1.time = t2.time " + + "order by COALESCE(time1, time2),device1,device2"; + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,null,d1,1,null,null,", + "null,2020-01-01T00:00:02.000Z,null,null,d1,20,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "null,2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,2020-01-01T00:00:05.000Z,d2,5,d2,50,", + "2020-01-01T00:00:07.000Z,null,d2,7,null,null,", + }; + expectedHeader = new String[] {"time1", "time2", "device1", "value1", "device2", "value2"}; + + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,null,d1,1,null,null,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:05.000Z,2020-01-01T00:00:05.000Z,d2,5,d2,50,", + "2020-01-01T00:00:07.000Z,null,d2,7,null,null,", + }; + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "null,2020-01-01T00:00:02.000Z,null,null,d1,20,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "null,2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,2020-01-01T00:00:05.000Z,d2,5,d2,50,", + }; + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:05.000Z,2020-01-01T00:00:05.000Z,d2,5,d2,50,", + }; + tableResultSetEqualTest(String.format(sql, INNER), expectedHeader, retArray, DATABASE_NAME); + + sql = + "select time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 from tableA t1 %s join tableB t2 USING(time) " + + "order by time,device1,device2"; + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,null,null,", + "2020-01-01T00:00:02.000Z,null,null,d1,20,", + "2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,d2,5,d2,50,", + "2020-01-01T00:00:07.000Z,d2,7,null,null,", + }; + expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"}; + + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,null,null,", + "2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:05.000Z,d2,5,d2,50,", + "2020-01-01T00:00:07.000Z,d2,7,null,null,", + }; + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:02.000Z,null,null,d1,20,", + "2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,d2,5,d2,50,", + }; + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:05.000Z,d2,5,d2,50,", + }; + tableResultSetEqualTest(String.format(sql, INNER), expectedHeader, retArray, DATABASE_NAME); + + // empty table join non-empty table + sql = + "select time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 from tableC t1 %s join tableB t2 USING(time) " + + "order by time,device1,device2"; + retArray = + new String[] { + "2020-01-01T00:00:02.000Z,null,null,d1,20,", + "2020-01-01T00:00:03.000Z,null,null,d1,30,", + "2020-01-01T00:00:03.000Z,null,null,d333,333,", + "2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,null,null,d2,50,", + }; + expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"}; + + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + retArray = new String[] {}; + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:02.000Z,null,null,d1,20,", + "2020-01-01T00:00:03.000Z,null,null,d1,30,", + "2020-01-01T00:00:03.000Z,null,null,d333,333,", + "2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,null,null,d2,50,", + }; + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + retArray = new String[] {}; + tableResultSetEqualTest(String.format(sql, INNER), expectedHeader, retArray, DATABASE_NAME); + + sql = + "select time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 " + + "from (select * from tableA where device='d1') t1 %s join (select * from tableB where device='d2') t2 USING(time) order by time, device1, device2"; + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,null,null,", + "2020-01-01T00:00:03.000Z,d1,3,null,null,", + "2020-01-01T00:00:04.000Z,null,null,d2,40,", + "2020-01-01T00:00:05.000Z,null,null,d2,50,", + }; + expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"}; + + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:01.000Z,d1,1,null,null,", "2020-01-01T00:00:03.000Z,d1,3,null,null,", + }; + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2020-01-01T00:00:04.000Z,null,null,d2,40,", "2020-01-01T00:00:05.000Z,null,null,d2,50,", + }; + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + retArray = new String[] {}; + tableResultSetEqualTest(String.format(sql, INNER), expectedHeader, retArray, DATABASE_NAME); + } + + // has filter + @Test + public void timeColumnOuterJoinTest2() { + expectedHeader = + new String[] {"time", "device", "level", "t1_num_add", "device", "attr2", "num", "str"}; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,null,null,null,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,null,null,null,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,null,null,null,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,null,null,null,d2,null,2,pineapple,", + "1970-01-01T00:00:00.040Z,null,null,null,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,null,null,null,d2,null,1,apricot,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.100Z,d1,l5,9,null,null,null,null,", + "1970-01-01T00:00:00.100Z,d2,l5,9,null,null,null,null,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:10.000Z,d1,l5,8,null,null,null,null,", + "1971-01-01T00:00:10.000Z,d2,l5,8,null,null,null,null,", + "1971-04-26T17:46:40.000Z,d1,l2,13,null,null,null,null,", + "1971-04-26T17:46:40.000Z,d2,l2,13,null,null,null,null,", + "1971-04-26T17:46:40.020Z,d1,l3,15,null,null,null,null,", + "1971-04-26T17:46:40.020Z,d2,l3,15,null,null,null,null,", + "1971-04-26T18:01:40.000Z,d1,l4,14,null,null,null,null,", + "1971-04-26T18:01:40.000Z,d2,l4,14,null,null,null,null,", + "1971-08-20T11:33:20.000Z,d1,l5,16,null,null,null,null,", + "1971-08-20T11:33:20.000Z,d2,l5,16,null,null,null,null,", + }; + + // join using + String sql1 = + "SELECT time, t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE TIME>=80 AND level!='l1' AND cast(num as double)>0) t1 \n" + + "%s JOIN (SELECT * FROM table0 WHERE TIME<=31536001000 AND floatNum<1000 AND device in ('d1','d2')) t2 \n" + + "USING(time) \n" + + "ORDER BY time, t1.device, t2.device"; + tableResultSetEqualTest(String.format(sql1, FULL), expectedHeader, retArray, DATABASE_NAME); + + // join on + String sql2 = + "SELECT COALESCE(t1.time, t2.time) as time, t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE TIME>=80 AND level!='l1' AND cast(num as double)>0) t1 \n" + + "%s JOIN (SELECT * FROM table0 WHERE TIME<=31536001000 AND floatNum<1000 AND device in ('d1','d2')) t2 \n" + + "ON t1.time = t2.time \n" + + "ORDER BY time, t1.device, t2.device"; + tableResultSetEqualTest(String.format(sql2, FULL), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "1970-01-01T00:00:00.080Z,d1,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.100Z,d1,l5,9,null,null,null,null,", + "1970-01-01T00:00:00.100Z,d2,l5,9,null,null,null,null,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:10.000Z,d1,l5,8,null,null,null,null,", + "1971-01-01T00:00:10.000Z,d2,l5,8,null,null,null,null,", + "1971-04-26T17:46:40.000Z,d1,l2,13,null,null,null,null,", + "1971-04-26T17:46:40.000Z,d2,l2,13,null,null,null,null,", + "1971-04-26T17:46:40.020Z,d1,l3,15,null,null,null,null,", + "1971-04-26T17:46:40.020Z,d2,l3,15,null,null,null,null,", + "1971-04-26T18:01:40.000Z,d1,l4,14,null,null,null,null,", + "1971-04-26T18:01:40.000Z,d2,l4,14,null,null,null,null,", + "1971-08-20T11:33:20.000Z,d1,l5,16,null,null,null,null,", + "1971-08-20T11:33:20.000Z,d2,l5,16,null,null,null,null,", + }; + tableResultSetEqualTest(String.format(sql1, LEFT), expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest(String.format(sql2, LEFT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,null,null,null,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,null,null,null,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,null,null,null,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,null,null,null,d2,null,2,pineapple,", + "1970-01-01T00:00:00.040Z,null,null,null,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,null,null,null,d2,null,1,apricot,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d2,null,9,apple,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d2,null,5,orange,", + }; + tableResultSetEqualTest(String.format(sql1, RIGHT), expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest(String.format(sql2, RIGHT), expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void fourTableJoinTest() { + expectedHeader = + new String[] { + "time", "s_tag", "s_name", "s_birth", "t_tag", "t_c_tag", "c_name", "g_tag", "score" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,1,Lucy,2015-10-10,1001,10000001,数学,1111,99,", + "1970-01-01T00:00:00.002Z,2,Jack,2015-09-24,1002,10000002,语文,1112,90,", + "1970-01-01T00:00:00.003Z,3,Sam,2014-07-20,1003,10000003,英语,1113,85,", + "1970-01-01T00:00:00.004Z,4,Lily,2015-03-28,1004,10000004,体育,1114,89,", + "1970-01-01T00:00:00.005Z,5,Helen,2016-01-22,1005,10000005,历史,1115,98,", + }; + sql = + "select s.time," + + " s.student_tag as s_tag, s.name as s_name, s.date_of_birth as s_birth," + + " t.teacher_tag as t_tag, t.course_tag as t_c_tag," + + " c.course_name as c_name," + + " g.grade_tag as g_tag, g.score as score " + + "from students s, teachers t, courses c, grades g " + + "where s.time=t.time AND c.time=g.time AND s.time=c.time " + + "order by s.student_tag, t.teacher_tag, c.course_tag,g.grade_tag"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"region", "name", "teacher_tag", "course_name", "score"}; + + retArray = + new String[] { + "haidian,Lucy,1005,数学,99,", + }; + sql = + "select s.region, s.name," + + " t.teacher_tag," + + " c.course_name," + + " g.score " + + "from students s, teachers t, courses c, grades g " + + "where s.time=c.time and c.time=g.time and t.teacher_tag = 1005 limit 1"; + + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void twoTableTimeColumnInnerJoinTest() { + expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"}; + sql = + "SELECT " + + " t1.time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 " + + "FROM " + + " tableA t1 JOIN tableB t2 " + + "ON t1.time = t2.time order by t1.time, device1, device2"; + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,d1,30,", + "2020-01-01T00:00:03.000Z,d1,3,d333,333,", + "2020-01-01T00:00:05.000Z,d2,5,d2,50,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void innerJoinOnMultiColumns() { + expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"}; + sql = + "SELECT " + + " t1.time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 " + + "FROM " + + " tableA t1, tableB t2 " + + "where t1.time = t2.time and t1.device = t2.device order by t1.time"; + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,d1,30,", "2020-01-01T00:00:05.000Z,d2,5,d2,50,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT " + + " t1.time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 " + + "FROM " + + " tableA t1 cross join tableB t2 " + + "where t1.time = t2.time and t1.device = t2.device order by t1.time"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT " + + " t1.time, " + + " t1.device as device1, " + + " t1.value as value1, " + + " t2.device as device2, " + + " t2.value as value2 " + + "FROM " + + " tableA t1 JOIN tableB t2 " + + "ON t1.time = t2.time and t1.device = t2.device order by t1.time"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device", "value1", "value2"}; + sql = + "SELECT " + + " time, device, " + + " t1.value as value1, " + + " t2.value as value2 " + + "FROM " + + " tableA t1 JOIN tableB t2 " + + "USING(time, device) ORDER BY time"; + retArray = + new String[] { + "2020-01-01T00:00:03.000Z,d1,3,30,", "2020-01-01T00:00:05.000Z,d2,5,50,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"device", "device", "num", "num", "floatnum", "floatnum"}; + sql = + "select t0.device, t1.device, t0.num, t1.num, t0.floatnum, t1.floatnum from table0 t0 join table1 t1 on t0.device=t1.device AND t0.attr2=t1.attr2 AND t0.num>t1.num AND t0.floatnum>t1.floatnum ORDER BY t0.device,t1.device,t0.num,t1.num"; + retArray = + new String[] { + "d1,d1,4,1,213.1,12.123,", "d1,d1,6,3,1231.21,231.2121,", "d1,d1,14,1,231.34,12.123,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time1", "time2", "device1", "device2"}; + sql = + "select t1.time as time1, t2.time as time2, t1.device as device1, t2.device as device2 from tablea t1 join tableb t2 " + + "on cast(substring(t1.device,2) as int32) = cast(substring(t2.device,2) as int32)+1 order by time1,time2,device1,device2"; + retArray = + new String[] { + "2020-01-01T00:00:05.000Z,2020-01-01T00:00:02.000Z,d2,d1,", + "2020-01-01T00:00:05.000Z,2020-01-01T00:00:03.000Z,d2,d1,", + "2020-01-01T00:00:07.000Z,2020-01-01T00:00:02.000Z,d2,d1,", + "2020-01-01T00:00:07.000Z,2020-01-01T00:00:03.000Z,d2,d1,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"attr1", "attr2"}; + sql = + "select t0.attr1,t0.attr2 from table0 t0 join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2"; + retArray = + new String[] { + "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", + "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "t,a,", + "t,a,", "t,a,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // expectedHeader = new String[] {"device", "device", "attr1", "attr1", "date", "date"}; + // sql = + // "select t0.device, t1.device, t0.attr1, t1.attr1, t0.date, t1.date from table0 t0 join + // table1 t1 on t0.device=t1.device and t0.attr1=t1.attr1 OR t0.date=t1.date"; + // retArray = + // new String[] { + // "d1,d1,c,c,null,2023-01-01,", + // "d1,d1,c,c,null,2023-01-01,", + // "d1,d1,c,c,null,2023-01-01,", + // "d1,d1,c,c,null,null,", + // "d1,d1,c,c,null,null,", + // "d1,d1,c,c,null,null,", + // "d1,d1,t,t,null,null,", + // "d1,d1,t,t,null,null,", + // "d1,d1,t,t,null,null,", + // "d2,d1,null,c,2023-01-01,2023-01-01,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void outerJoinTest() { + expectedHeader = new String[] {"date", "date"}; + sql = + "select t0.date, t1.date from table0 t0 %s join table1 t1 on t0.date=t1.date order by t0.date, t1.date"; + retArray = + new String[] { + "2022-01-01,null,", + "2023-01-01,2023-01-01,", + "null,2023-05-01,", + "null,2023-10-01,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + }; + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + retArray = + new String[] { + "2022-01-01,null,", + "2023-01-01,2023-01-01,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null," + }; + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + retArray = + new String[] { + "2023-01-01,2023-01-01,", + "null,2023-05-01,", + "null,2023-10-01,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + "null,null,", + }; + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"attr1", "attr2", "attr1", "attr2"}; + sql = + "select t0.attr1,t0.attr2,t1.attr1,t1.attr2 from table0 t0 %s join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2 order by t0.attr1,t0.attr2,t1.attr1,t1.attr2"; + retArray = + new String[] { + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "d,c,null,null,", + "d,c,null,null,", + "d,c,null,null,", + "t,a,t,a,", + "t,a,t,a,", + "t,a,t,a,", + "vv,null,null,null,", + "vv,null,null,null,", + "vv,null,null,null,", + "yy,zz,null,null,", + "yy,zz,null,null,", + "yy,zz,null,null,", + "null,null,y,z,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + }; + tableResultSetEqualTest(String.format(sql, FULL), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "d,c,null,null,", + "d,c,null,null,", + "d,c,null,null,", + "t,a,t,a,", + "t,a,t,a,", + "t,a,t,a,", + "vv,null,null,null,", + "vv,null,null,null,", + "vv,null,null,null,", + "yy,zz,null,null,", + "yy,zz,null,null,", + "yy,zz,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + }; + tableResultSetEqualTest(String.format(sql, LEFT), expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "c,d,c,d,", + "t,a,t,a,", + "t,a,t,a,", + "t,a,t,a,", + "null,null,y,z,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null,", + "null,null,null,null," + }; + tableResultSetEqualTest(String.format(sql, RIGHT), expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void lastCacheTest() { + expectedHeader = + new String[] { + "level", "attr1", "device", "attr2", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13", "_col14", "_col15" + }; + + // last query without filter + retArray = + new String[] { + "l1,c,d1,d,1971-01-01T00:01:40.000Z,d1,l1,c,d,1971-01-01T00:01:40.000Z,11,2147468648,54.121,1971-01-01T00:01:40.000Z,d1,11,", + "l2,yy,d1,zz,1971-04-26T17:46:40.000Z,d1,l2,yy,zz,1971-04-26T17:46:40.000Z,12,2146483648,45.231,1971-04-26T17:46:40.000Z,d1,12,", + "l3,t,d1,a,1971-04-26T17:46:40.020Z,d1,l3,t,a,1971-04-26T17:46:40.020Z,14,2907483648,231.34,1971-04-26T17:46:40.020Z,d1,14,", + "l4,null,d1,null,1971-04-26T18:01:40.000Z,d1,l4,null,null,1971-04-26T18:01:40.000Z,13,2107483648,54.12,1971-04-26T18:01:40.000Z,d1,13,", + "l5,null,d1,null,1971-08-20T11:33:20.000Z,d1,l5,null,null,1971-08-20T11:33:20.000Z,15,3147483648,235.213,1971-08-20T11:33:20.000Z,d1,15,", + "l1,d,d2,c,1971-01-01T00:01:40.000Z,d2,l1,d,c,1971-01-01T00:01:40.000Z,11,2147468648,54.121,1971-01-01T00:01:40.000Z,d2,11,", + "l2,vv,d2,null,1971-04-26T17:46:40.000Z,d2,l2,vv,null,1971-04-26T17:46:40.000Z,12,2146483648,45.231,1971-04-26T17:46:40.000Z,d2,12,", + "l3,null,d2,null,1971-04-26T17:46:40.020Z,d2,l3,null,null,1971-04-26T17:46:40.020Z,14,2907483648,231.34,1971-04-26T17:46:40.020Z,d2,14,", + "l4,null,d2,null,1971-04-26T18:01:40.000Z,d2,l4,null,null,1971-04-26T18:01:40.000Z,13,2107483648,54.12,1971-04-26T18:01:40.000Z,d2,13,", + "l5,null,d2,null,1971-08-20T11:33:20.000Z,d2,l5,null,null,1971-08-20T11:33:20.000Z,15,3147483648,235.213,1971-08-20T11:33:20.000Z,d2,15,", + }; + sql = + "select level, attr1, device, attr2, last_by(time,time),last_by(device,time),last_by(level,time),last_by(attr1,time),last_by(attr2,time),last(time),last_by(num,time),last_by(bignum,time),last_by(floatnum,time),last_by(time,time),last_by(device,time),last_by(num,time) from table0 group by attr1, device, attr2, level order by device,level,attr1,attr2"; + repeatTest(sql, expectedHeader, retArray, DATABASE_NAME, 3); + + // last query with lt time filter + retArray = + new String[] { + "l1,c,d1,d,1971-01-01T00:01:40.000Z,d1,l1,c,d,1971-01-01T00:01:40.000Z,11,2147468648,54.121,1971-01-01T00:01:40.000Z,d1,11,", + "l2,yy,d1,zz,1971-01-01T00:00:00.100Z,d1,l2,yy,zz,1971-01-01T00:00:00.100Z,10,3147483648,231.55,1971-01-01T00:00:00.100Z,d1,10,", + "l3,t,d1,a,1971-01-01T00:00:00.500Z,d1,l3,t,a,1971-01-01T00:00:00.500Z,4,2147493648,213.1,1971-01-01T00:00:00.500Z,d1,4,", + "l4,null,d1,null,1971-01-01T00:00:01.000Z,d1,l4,null,null,1971-01-01T00:00:01.000Z,5,2149783648,56.32,1971-01-01T00:00:01.000Z,d1,5,", + "l5,null,d1,null,1971-01-01T00:00:10.000Z,d1,l5,null,null,1971-01-01T00:00:10.000Z,7,2147983648,213.112,1971-01-01T00:00:10.000Z,d1,7,", + "l1,d,d2,c,1971-01-01T00:01:40.000Z,d2,l1,d,c,1971-01-01T00:01:40.000Z,11,2147468648,54.121,1971-01-01T00:01:40.000Z,d2,11,", + "l2,vv,d2,null,1971-01-01T00:00:00.100Z,d2,l2,vv,null,1971-01-01T00:00:00.100Z,10,3147483648,231.55,1971-01-01T00:00:00.100Z,d2,10,", + "l3,null,d2,null,1971-01-01T00:00:00.500Z,d2,l3,null,null,1971-01-01T00:00:00.500Z,4,2147493648,213.1,1971-01-01T00:00:00.500Z,d2,4,", + "l4,null,d2,null,1971-01-01T00:00:01.000Z,d2,l4,null,null,1971-01-01T00:00:01.000Z,5,2149783648,56.32,1971-01-01T00:00:01.000Z,d2,5,", + "l5,null,d2,null,1971-01-01T00:00:10.000Z,d2,l5,null,null,1971-01-01T00:00:10.000Z,7,2147983648,213.112,1971-01-01T00:00:10.000Z,d2,7,", + }; + sql = + "select level, attr1, device, attr2, last_by(time,time),last_by(device,time),last_by(level,time),last_by(attr1,time),last_by(attr2,time),last(time),last_by(num,time),last_by(bignum,time),last_by(floatnum,time),last_by(time,time),last_by(device,time),last_by(num,time) from table0 where time<1971-04-26T17:46:40.000 group by attr1, device, attr2, level order by device,level,attr1,attr2"; + repeatTest(sql, expectedHeader, retArray, DATABASE_NAME, 3); + + // last query with gt time filter + retArray = + new String[] { + "l3,t,d1,a,1971-04-26T17:46:40.020Z,d1,l3,t,a,1971-04-26T17:46:40.020Z,14,2907483648,231.34,1971-04-26T17:46:40.020Z,d1,14,", + "l4,null,d1,null,1971-04-26T18:01:40.000Z,d1,l4,null,null,1971-04-26T18:01:40.000Z,13,2107483648,54.12,1971-04-26T18:01:40.000Z,d1,13,", + "l5,null,d1,null,1971-08-20T11:33:20.000Z,d1,l5,null,null,1971-08-20T11:33:20.000Z,15,3147483648,235.213,1971-08-20T11:33:20.000Z,d1,15,", + "l3,null,d2,null,1971-04-26T17:46:40.020Z,d2,l3,null,null,1971-04-26T17:46:40.020Z,14,2907483648,231.34,1971-04-26T17:46:40.020Z,d2,14,", + "l4,null,d2,null,1971-04-26T18:01:40.000Z,d2,l4,null,null,1971-04-26T18:01:40.000Z,13,2107483648,54.12,1971-04-26T18:01:40.000Z,d2,13,", + "l5,null,d2,null,1971-08-20T11:33:20.000Z,d2,l5,null,null,1971-08-20T11:33:20.000Z,15,3147483648,235.213,1971-08-20T11:33:20.000Z,d2,15,", + }; + sql = + "select level, attr1, device, attr2, last_by(time,time),last_by(device,time),last_by(level,time),last_by(attr1,time),last_by(attr2,time),last(time)," + + "last_by(num,time),last_by(bignum,time),last_by(floatnum,time),last_by(time,time),last_by(device,time),last_by(num,time) from table0 where time>1971-04-26T17:46:40.000 group by attr1, device, attr2, level order by device,level,attr1,attr2"; + repeatTest(sql, expectedHeader, retArray, DATABASE_NAME, 3); + } + + @Test + public void asofInnerJoinTest() { + expectedHeader = new String[] {"time", "device", "level", "time", "device", "level"}; + retArray = + new String[] { + "1971-01-01T00:00:00.000Z,d1,l1,1970-01-01T00:00:00.100Z,d1,l5,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,d999,null,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,null,l999,", + "1970-01-01T00:00:00.020Z,d1,l2,1970-01-01T00:00:00.010Z,d11,l11,", + "1971-01-01T00:00:00.100Z,d1,l2,1971-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:00:00.100Z,d1,l2,1971-01-01T00:00:00.000Z,d999,null,", + "1971-01-01T00:00:00.100Z,d1,l2,1971-01-01T00:00:00.000Z,null,l999,", + "1971-04-26T17:46:40.000Z,d1,l2,1971-01-01T00:00:00.000Z,d1,l1,", + "1971-04-26T17:46:40.000Z,d1,l2,1971-01-01T00:00:00.000Z,d999,null," + }; + // test single join condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof join table1 on " + + "table0.time>table1.time " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + // test expr and '>=' in ASOF condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof join table1 on " + + "table0.time>=table1.time+1 " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1971-01-01T00:00:00.000Z,d1,l1,1970-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:00:00.100Z,d1,l2,1970-01-01T00:00:00.020Z,d1,l2,", + "1971-04-26T17:46:40.000Z,d1,l2,1970-01-01T00:00:00.020Z,d1,l2,", + "1971-01-01T00:00:00.500Z,d1,l3,1970-01-01T00:00:00.040Z,d1,l3,", + "1971-04-26T17:46:40.020Z,d1,l3,1970-01-01T00:00:00.040Z,d1,l3,", + "1971-01-01T00:00:01.000Z,d1,l4,1970-01-01T00:00:00.080Z,d1,l4,", + "1971-04-26T18:01:40.000Z,d1,l4,1970-01-01T00:00:00.080Z,d1,l4,", + "1971-01-01T00:00:10.000Z,d1,l5,1970-01-01T00:00:00.100Z,d1,l5,", + "1971-08-20T11:33:20.000Z,d1,l5,1970-01-01T00:00:00.100Z,d1,l5," + }; + // test multi join conditions + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof join table1 on " + + "table0.device=table1.device and table1.level=table0.level and table0.time>table1.time " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level", + expectedHeader, + retArray, + DATABASE_NAME); + // test expr and '>=' in ASOF condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof join table1 on " + + "table0.device=table1.device and table1.level=table0.level and table0.time>=table1.time+1 " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,l1,1970-01-01T00:00:00.010Z,d11,l11,", + "1970-01-01T00:00:00.020Z,d1,l2,1970-01-01T00:00:00.030Z,d11,l11,", + "1970-01-01T00:00:00.040Z,d1,l3,1970-01-01T00:00:00.080Z,d1,l4,", + "1970-01-01T00:00:00.080Z,d1,l4,1970-01-01T00:00:00.100Z,d1,l5,", + "1970-01-01T00:00:00.100Z,d1,l5,1971-01-01T00:00:00.000Z,d1,l1,", + "1970-01-01T00:00:00.100Z,d1,l5,1971-01-01T00:00:00.000Z,d999,null,", + "1970-01-01T00:00:00.100Z,d1,l5,1971-01-01T00:00:00.000Z,null,l999,", + "1970-01-01T00:00:00.000Z,d2,l1,1970-01-01T00:00:00.010Z,d11,l11,", + "1970-01-01T00:00:00.020Z,d2,l2,1970-01-01T00:00:00.030Z,d11,l11,", + "1970-01-01T00:00:00.040Z,d2,l3,1970-01-01T00:00:00.080Z,d1,l4," + }; + // test single join condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof join table1 on " + + "table0.time=' in ASOF condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof join table1 on " + + "table0.device=table1.device and table1.level=table0.level and table0.time<=table1.time-1 " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void asofLeftJoinTest() { + expectedHeader = new String[] {"time", "device", "level", "time", "device", "level"}; + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,l1,null,null,null,", + "1971-01-01T00:00:00.000Z,d1,l1,1970-01-01T00:00:00.100Z,d1,l5,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,d999,null,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,null,l999,", + "1970-01-01T00:00:00.020Z,d1,l2,1970-01-01T00:00:00.010Z,d11,l11,", + "1971-01-01T00:00:00.100Z,d1,l2,1971-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:00:00.100Z,d1,l2,1971-01-01T00:00:00.000Z,d999,null,", + "1971-01-01T00:00:00.100Z,d1,l2,1971-01-01T00:00:00.000Z,null,l999,", + "1971-04-26T17:46:40.000Z,d1,l2,1971-01-01T00:00:00.000Z,d1,l1," + }; + // test single join condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof left join table1 on " + + "table0.time>table1.time " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + // test expr and '>=' in ASOF condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof left join table1 on " + + "table0.time>=table1.time+1 " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,l1,null,null,null,", + "1971-01-01T00:00:00.000Z,d1,l1,1970-01-01T00:00:00.000Z,d1,l1,", + "1971-01-01T00:01:40.000Z,d1,l1,1971-01-01T00:00:00.000Z,d1,l1,", + "1970-01-01T00:00:00.020Z,d1,l2,null,null,null,", + "1971-01-01T00:00:00.100Z,d1,l2,1970-01-01T00:00:00.020Z,d1,l2,", + "1971-04-26T17:46:40.000Z,d1,l2,1970-01-01T00:00:00.020Z,d1,l2,", + "1970-01-01T00:00:00.040Z,d1,l3,null,null,null,", + "1971-01-01T00:00:00.500Z,d1,l3,1970-01-01T00:00:00.040Z,d1,l3,", + "1971-04-26T17:46:40.020Z,d1,l3,1970-01-01T00:00:00.040Z,d1,l3,", + "1970-01-01T00:00:00.080Z,d1,l4,null,null,null," + }; + // test multi join conditions + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof left join table1 on " + + "table0.device=table1.device and table1.level=table0.level and table0.time>table1.time " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + // test expr and '>=' in ASOF condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof left join table1 on " + + "table0.device=table1.device and table1.level=table0.level and table0.time>=table1.time+1 " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,l1,1970-01-01T00:00:00.010Z,d11,l11,", + "1971-01-01T00:00:00.000Z,d1,l1,null,null,null,", + "1971-01-01T00:01:40.000Z,d1,l1,null,null,null,", + "1970-01-01T00:00:00.020Z,d1,l2,1970-01-01T00:00:00.030Z,d11,l11,", + "1971-01-01T00:00:00.100Z,d1,l2,null,null,null,", + "1971-04-26T17:46:40.000Z,d1,l2,null,null,null,", + "1970-01-01T00:00:00.040Z,d1,l3,1970-01-01T00:00:00.080Z,d1,l4,", + "1971-01-01T00:00:00.500Z,d1,l3,null,null,null,", + "1971-04-26T17:46:40.020Z,d1,l3,null,null,null,", + "1970-01-01T00:00:00.080Z,d1,l4,1970-01-01T00:00:00.100Z,d1,l5," + }; + // test single join condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof left join table1 on " + + "table0.time=' in ASOF condition + tableResultSetEqualTest( + "select table0.time,table0.device,table0.level,table1.time,table1.device,table1.level from table0 asof left join table1 on " + + "table0.device=table1.device and table1.level=table0.level and table0.time<=table1.time-1 " + + "order by table0.device,table0.level,table0.time,table1.device,table1.level limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest() { + String errMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": " + ONLY_SUPPORT_EQUI_JOIN; + tableAssertTestFail( + "select * from table0 t0 full join table1 t1 on t0.num>t1.num", errMsg, DATABASE_NAME); + + tableAssertTestFail( + "select * from table0 t0 full join table1 t1 on t0.num!=t1.num", errMsg, DATABASE_NAME); + + tableAssertTestFail( + "select * from table0 t0 full join table1 t1 on t0.device=t1.device AND t0.num>t1.num", + errMsg, + DATABASE_NAME); + + tableAssertTestFail( + "select * from table0 t0 full join table1 t1 on t0.device=t1.device OR t0.num>t1.num", + errMsg, + DATABASE_NAME); + + tableAssertTestFail( + "select * from table0 t0 full join table1 t1 on t0.device=t1.device OR t0.time=t1.time", + errMsg, + DATABASE_NAME); + + tableAssertTestFail( + "select * from table0 asof (tolerance 1s) left join table1 on table0.time<=table1.time", + "Tolerance in ASOF JOIN is only support INNER type now", + DATABASE_NAME); + } + + @Test + public void aggregationTableScanWithJoinTest() { + expectedHeader = new String[] {"date", "_col1", "date", "_col3"}; + retArray = new String[] {"1970-01-01T00:00:00.000Z,2,1970-01-01T00:00:00.000Z,2,"}; + // Join may rename the 'time' column, so we need to ensure the correctness of + // AggregationTableScan in this case + tableResultSetEqualTest( + "select * from (" + + "select date_bin(1ms,time) as date,count(*)from table0 group by date_bin(1ms,time)) t0 " + + "join (" + + "select date_bin(1ms,time) as date,count(*)from table1 where time=0 group by date_bin(1ms,time)) t1 " + + "on t0.date = t1.date", + expectedHeader, + retArray, + DATABASE_NAME); + } + + public static void repeatTest( + String sql, String[] expectedHeader, String[] retArray, String dbName, int repeatTimes) { + for (int i = 0; i < repeatTimes; i++) { + tableResultSetEqualTest(sql, expectedHeader, retArray, dbName); + } + } + + public static String[] buildHeaders(int length) { + String[] expectedHeader = new String[length]; + for (int i = 0; i < length; i++) { + expectedHeader[i] = "_col" + i; + } + return expectedHeader; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRecoverTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRecoverTableIT.java new file mode 100644 index 0000000000000..8a9c9bc4c5b74 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRecoverTableIT.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Locale; + +import static org.apache.iotdb.db.utils.constant.TestConstant.count; +import static org.apache.iotdb.db.utils.constant.TestConstant.maxValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.minTime; +import static org.apache.iotdb.db.utils.constant.TestConstant.minValue; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +@Ignore // aggregation +public class IoTDBRecoverTableIT { + + private static final Logger logger = LoggerFactory.getLogger(IoTDBRecoverTableIT.class); + + private static final String TIMESTAMP_STR = "Time"; + private static final String TEMPERATURE_STR = "temperature"; + private static final String[] creationSqls = + new String[] { + "CREATE DATABASE test", + "USE \"test\"", + "CREATE TABLE vehicle (tag1 string tag, s0 int32 field, s1 int64 field, s2 float field, s3 text field, s4 boolean field)" + }; + private static final String[] dataSet2 = + new String[] { + "CREATE DATABASE ln", + "USE \"ln\"", + "CREATE TABLE wf01 (tag1 string tag, status boolean field, temperature float field, hardware int32 field)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 1, 1.1, false, 11)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 2, 2.2, true, 22)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 3, 3.3, false, 33 )", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 4, 4.4, false, 44)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01',5, 5.5, false, 55)" + }; + private static final String d0s0 = "s0"; + private static final String d0s1 = "s1"; + private static final String d0s2 = "s2"; + private static final String d0s3 = "s3"; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void recoverTest1() { + // stop cluster + EnvFactory.getEnv().shutdownAllDataNodes(); + logger.info("All DataNodes are shut down"); + EnvFactory.getEnv().shutdownAllConfigNodes(); + logger.info("All ConfigNodes are shut down"); + EnvFactory.getEnv().startAllConfigNodes(); + logger.info("All ConfigNodes are started"); + EnvFactory.getEnv().startAllDataNodes(); + logger.info("All DataNodes are started"); + // check cluster whether restart + ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); + String[] retArray = new String[] {"0,2", "0,4", "0,3"}; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("use \"ln\""); + String selectSql = "select count(temperature) from wf01 where time > 3"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + "," + resultSet.getString(count(TEMPERATURE_STR)); + Assert.assertEquals(retArray[0], ans); + } + + selectSql = "select min_time(temperature) from wf01 where time > 3"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + "," + resultSet.getString(minTime(TEMPERATURE_STR)); + Assert.assertEquals(retArray[1], ans); + } + + selectSql = "select min_time(temperature) from wf01 where temperature > 3"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + "," + resultSet.getString(minTime(TEMPERATURE_STR)); + Assert.assertEquals(retArray[2], ans); + } + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // max min ValueTest + retArray = new String[] {"0,8499,500.0", "0,2499,500.0"}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + String selectSql = + "select max_value(s0),min_value(s2) " + + "from root.vehicle.d0 where time >= 100 and time < 9000"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString(maxValue(d0s0)) + + "," + + resultSet.getString(minValue(d0s2)); + Assert.assertEquals(retArray[0], ans); + } + + selectSql = "select max_value(s0),min_value(s2) from root.vehicle.d0 where time < 2500"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString(maxValue(d0s0)) + + "," + + resultSet.getString(minValue(d0s2)); + Assert.assertEquals(retArray[1], ans); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void recoverTest2() { + // stop cluster + EnvFactory.getEnv().shutdownAllDataNodes(); + logger.info("All DataNodes are shut down"); + EnvFactory.getEnv().shutdownAllConfigNodes(); + logger.info("All ConfigNodes are shut down"); + EnvFactory.getEnv().startAllConfigNodes(); + logger.info("All ConfigNodes are started"); + EnvFactory.getEnv().startAllDataNodes(); + logger.info("All DataNodes are started"); + // wait for cluster to start and check + ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); + // count test + String[] retArray = new String[] {"0,2001,2001,2001,2001", "0,7500,7500,7500,7500"}; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + String selectSql = + "select count(s0),count(s1),count(s2),count(s3) " + + "from vehicle where time >= 6000 and time <= 9000"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString(count(d0s0)) + + "," + + resultSet.getString(count(d0s1)) + + "," + + resultSet.getString(count(d0s2)) + + "," + + resultSet.getString(count(d0s3)); + Assert.assertEquals(retArray[0], ans); + } + + selectSql = "select count(s0),count(s1),count(s2),count(s3) from vehicle"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString(count(d0s0)) + + "," + + resultSet.getString(count(d0s1)) + + "," + + resultSet.getString(count(d0s2)) + + "," + + resultSet.getString(count(d0s3)); + Assert.assertEquals(retArray[1], ans); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private static void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.execute(sql); + } + + for (String sql : dataSet2) { + statement.execute(sql); + } + + // prepare BufferWrite file + String insertTemplate = + "INSERT INTO vehicle(tag1,timestamp,s0,s1,s2,s3,s4)" + " VALUES('d0',%d,%d,%d,%f,%s,%s)"; + for (int i = 5000; i < 7000; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); + } + statement.executeBatch(); + statement.execute("flush"); + for (int i = 7500; i < 8500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + } + statement.executeBatch(); + statement.execute("flush"); + // prepare Unseq-File + for (int i = 500; i < 1500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); + } + statement.executeBatch(); + statement.execute("flush"); + for (int i = 3000; i < 6500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + } + statement.executeBatch(); + statement.execute("flush"); + + // prepare BufferWrite cache + for (int i = 9000; i < 10000; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); + } + statement.executeBatch(); + // prepare Overflow cache + for (int i = 2000; i < 2500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + } + statement.executeBatch(); + statement.execute("flush"); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRecoverUnclosedTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRecoverUnclosedTableIT.java new file mode 100644 index 0000000000000..ce3b4bd916172 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRecoverUnclosedTableIT.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.apache.iotdb.db.utils.constant.TestConstant.count; +import static org.apache.iotdb.db.utils.constant.TestConstant.maxValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.minTime; +import static org.apache.iotdb.db.utils.constant.TestConstant.minValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +@Ignore // aggregation +public class IoTDBRecoverUnclosedTableIT { + private static final Logger logger = LoggerFactory.getLogger(IoTDBRecoverUnclosedTableIT.class); + private static final String TIMESTAMP_STR = "time"; + private static final String TEMPERATURE_STR = "temperature"; + private static final String[] creationSqls = + new String[] { + "CREATE DATABASE test", + "USE \"test\"", + "CREATE TABLE vehicle (tag1 string tag, s0 int32 field, s1 int64 field, s2 float field, s3 text field, s4 boolean field)" + }; + private static final String[] dataSet2 = + new String[] { + "CREATE DATABASE ln", + "USE \"ln\"", + "CREATE TABLE wf01 (tag1 string tag, status boolean field, temperature float field, hardware int32 field)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 1, 1.1, false, 11)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 2, 2.2, true, 22)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 3, 3.3, false, 33 )", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01', 4, 4.4, false, 44)", + "INSERT INTO wf01(tag1, time,temperature,status, hardware) " + + "values('wt01',5, 5.5, false, 55)" + }; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws SQLException, IOException { + String[] retArray = new String[] {"0,2", "0,4", "0,3"}; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("use \"ln\""); + String selectSql = "select count(temperature) from wf01 where time > 3"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + "," + resultSet.getString(count(TEMPERATURE_STR)); + assertEquals(retArray[0], ans); + } + + selectSql = "select min_time(temperature) from wf01 where time > 3"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + "," + resultSet.getString(minTime(TEMPERATURE_STR)); + assertEquals(retArray[1], ans); + } + + selectSql = "select min_time(temperature) from wf01 where temperature > 3"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + "," + resultSet.getString(minTime(TEMPERATURE_STR)); + assertEquals(retArray[2], ans); + } + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + insertMoreData(); + + // stop cluster + EnvFactory.getEnv().shutdownAllDataNodes(); + logger.info("All DataNodes are shut down"); + EnvFactory.getEnv().shutdownAllConfigNodes(); + logger.info("All ConfigNodes are shut down"); + EnvFactory.getEnv().startAllConfigNodes(); + logger.info("All ConfigNodes are started"); + EnvFactory.getEnv().startAllDataNodes(); + logger.info("All DataNodes are started"); + // wait for cluster to start and check + ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); + + // test count, + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + String selectSql = "select count(*) from vehicle"; + ResultSet tempResultSet = statement.executeQuery(selectSql); + assertNotNull(tempResultSet); + tempResultSet.next(); + String d0s0 = "s0"; + String d0s1 = "s1"; + String d0s2 = "s2"; + assertEquals(7500, tempResultSet.getInt("count(" + d0s0 + ")")); + + // test max, min value + retArray = new String[] {"0,8499,500.0", "0,2499,500.0"}; + selectSql = + "select max_value(s0),min_value(s2) " + "from vehicle where time >= 100 and time < 9000"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue(d0s0)) + + "," + + resultSet.getString(minValue(d0s2)); + assertEquals(retArray[0], ans); + } + + selectSql = "select max_value(s1),min_value(s2) from vehicle where time < 2500"; + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + assertNotNull(resultSet); + resultSet.next(); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString(maxValue(d0s1)) + + "," + + resultSet.getString(minValue(d0s2)); + assertEquals(retArray[1], ans); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + for (String sql : creationSqls) { + statement.execute(sql); + } + + for (String sql : dataSet2) { + statement.execute(sql); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void insertMoreData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // prepare BufferWrite file + String insertTemplate = + "INSERT INTO vehicle(tag1,timestamp,s0,s1,s2,s3,s4)" + " VALUES('d0',%d,%d,%d,%f,%s,%s)"; + for (int i = 5000; i < 7000; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); + } + statement.executeBatch(); + for (int i = 7500; i < 8500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + } + statement.executeBatch(); + // prepare Unseq-File + for (int i = 500; i < 1500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); + } + for (int i = 3000; i < 6500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + } + statement.executeBatch(); + // prepare BufferWrite cache + for (int i = 9000; i < 10000; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); + } + statement.executeBatch(); + // prepare Overflow cache + for (int i = 2000; i < 2500; i++) { + statement.addBatch( + String.format( + Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + } + statement.executeBatch(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java new file mode 100644 index 0000000000000..70efd0a91646b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRestartTableIT { + + private final Logger logger = LoggerFactory.getLogger(IoTDBRestartTableIT.class); + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.ratis.RatisConsensus"); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("ASYNC"); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus"); + } + + @Test + public void testRestart() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database test"); + statement.execute("use \"test\""); + statement.execute("create table turbine (tag1 string tag, s1 float field)"); + statement.execute("insert into turbine(tag1, time,s1) values('d1', 1,1.0)"); + statement.execute("flush"); + } + + try { + TestUtils.restartDataNodes(); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into turbine(tag1, time,s1) values('d1', 2,1.0)"); + } + + try { + TestUtils.restartDataNodes(); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into turbine(tag1, time,s1) values('d1', 3,1.0)"); + + String[] exp = new String[] {"1,1.0", "2,1.0", "3,1.0"}; + int cnt = 0; + try (ResultSet resultSet = statement.executeQuery("SELECT time, s1 FROM turbine")) { + assertNotNull(resultSet); + while (resultSet.next()) { + String result = resultSet.getLong("time") + "," + resultSet.getString(2); + assertEquals(exp[cnt], result); + cnt++; + } + } + } + } + + @Ignore // data deletion + @Test + public void testRestartDelete() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into root.turbine.d1(time,s1) values(1,1)"); + statement.execute("insert into root.turbine.d1(time,s1) values(2,2)"); + statement.execute("insert into root.turbine.d1(time,s1) values(3,3)"); + } + + TestUtils.restartDataNodes(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("delete from root.turbine.d1.s1 where time<=1"); + + ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.turbine.d1"); + assertNotNull(resultSet); + String[] exp = new String[] {"2,2.0", "3,3.0"}; + int cnt = 0; + try { + while (resultSet.next()) { + String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); + assertEquals(exp[cnt], result); + cnt++; + } + + statement.execute("flush"); + statement.execute("delete from root.turbine.d1.s1 where time<=2"); + + exp = new String[] {"3,3.0"}; + resultSet = statement.executeQuery("SELECT s1 FROM root.turbine.d1"); + assertNotNull(resultSet); + cnt = 0; + while (resultSet.next()) { + String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); + assertEquals(exp[cnt], result); + cnt++; + } + } finally { + resultSet.close(); + } + } + } + + @Test + public void testRestartQueryLargerThanEndTime() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database \"test\""); + statement.execute("use \"test\""); + statement.execute("create table turbine (tag1 string tag, s1 int64 field, s2 boolean field)"); + statement.execute("insert into turbine(time,tag1,s1) values(1,\'d1\',1)"); + statement.execute("insert into turbine(time,tag1,s1) values(2,\'d1\',2)"); + } + + try { + TestUtils.restartDataNodes(); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into turbine(time,tag1,s1) values(3,\'d1\',1)"); + statement.execute("insert into turbine(time,tag1,s1) values(4,\'d1\',2)"); + } + + try { + TestUtils.restartDataNodes(); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + + String[] exp = + new String[] { + "4,2", + }; + int cnt = 0; + try (ResultSet resultSet = + statement.executeQuery("SELECT Time, s1 FROM turbine where time > 3 and tag1=\'d1\'")) { + assertNotNull(resultSet); + while (resultSet.next()) { + String result = resultSet.getLong(TIMESTAMP_STR) + "," + resultSet.getString(2); + assertEquals(exp[cnt], result); + cnt++; + } + } + assertEquals(1, cnt); + } + } + + @Test + public void testRestartEndTime() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database \"test\""); + statement.execute("use \"test\""); + statement.execute("create table turbine (tag1 string tag, s1 int64 field, s2 int64 field)"); + statement.execute("insert into turbine(time,\"tag1\",s1) values(1,\'d1\',1)"); + statement.execute("insert into turbine(time,\"tag1\",s1) values(2,\'d1\',2)"); + } + + try { + TestUtils.restartDataNodes(); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into turbine(time,tag1,s2) values(1,\'d1\',1)"); + statement.execute("insert into turbine(time,tag1,s2) values(2,\'d1\',2)"); + } + + try { + TestUtils.restartDataNodes(); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + + String[] exp = new String[] {"1,1", "2,2"}; + int cnt = 0; + try (ResultSet resultSet = + statement.executeQuery("SELECT Time, s2 FROM turbine where tag1=\'d1\'")) { + assertNotNull(resultSet); + while (resultSet.next()) { + String result = resultSet.getLong(TIMESTAMP_STR) + "," + resultSet.getLong(2); + assertEquals(exp[cnt], result); + cnt++; + } + } + assertEquals(2, cnt); + } + } + + @Ignore // delete column + @Test + public void testRecoverWALMismatchDataType() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into root.turbine1.d1(time,s1,s2) values(1,1.1,2.2)"); + statement.execute("delete timeseries root.turbine1.d1.s1"); + statement.execute( + "create timeseries root.turbine1.d1.s1 with datatype=INT32, encoding=RLE, compression=SNAPPY"); + } + + TestUtils.restartDataNodes(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + + try (ResultSet resultSet = statement.executeQuery("select * from root.**")) { + assertNotNull(resultSet); + int cnt = 0; + assertEquals(3, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + assertEquals("1", resultSet.getString(1)); + assertNull(resultSet.getString(2)); + assertEquals("2.2", resultSet.getString(3)); + cnt++; + } + assertEquals(1, cnt); + } + } + } + + @Ignore // delete column + @Test + public void testRecoverWALDeleteSchema() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + statement.execute("insert into turbine1(time,tag1,s1,s2) values(1,\'d1\',1.1,2.2)"); + statement.execute("delete timeseries root.turbine1.d1.s1"); + } + + TestUtils.restartDataNodes(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + + try (ResultSet resultSet = statement.executeQuery("select * from root.**")) { + assertNotNull(resultSet); + int cnt = 0; + assertEquals(2, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + assertEquals("1", resultSet.getString(1)); + assertEquals("2.2", resultSet.getString(2)); + cnt++; + } + assertEquals(1, cnt); + } + } + } + + @Test + public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + long tsFileSize = config.getSeqTsFileSize(); + long unFsFileSize = config.getSeqTsFileSize(); + config.setSeqTsFileSize(10000000); + config.setUnSeqTsFileSize(10000000); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database \"test\""); + statement.execute("use \"test\""); + statement.execute( + "create table turbine1 (tag1 string tag, s1 int64 field, s2 boolean field)"); + statement.execute("insert into turbine1(time,tag1,s1) values(1,\'d1\',1)"); + statement.execute("insert into turbine1(time,tag1,s1) values(2,\'d1\',2)"); + statement.execute("insert into turbine1(time,tag1,s2) values(3,\'d1\',true)"); + statement.execute("insert into turbine1(time,tag1,s2) values(4,\'d1\',true)"); + } + + TestUtils.restartDataNodes(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + + long[] result = new long[] {1L, 2L}; + ResultSet resultSet = + statement.executeQuery("select s1 from turbine1 where time < 3 order by time"); + assertNotNull(resultSet); + int cnt = 0; + while (resultSet.next()) { + assertEquals(result[cnt], resultSet.getLong(1)); + cnt++; + } + assertEquals(2, cnt); + } + + config.setSeqTsFileSize(tsFileSize); + config.setUnSeqTsFileSize(unFsFileSize); + } + + @Test + public void testRecoverFromFlushMemTableError() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database \"test\""); + statement.execute("use \"test\""); + statement.execute("create table turbine1 (tag1 string tag, s1 float field, s2 double field)"); + statement.execute("insert into turbine1(time,tag1,s1,s2) values(1,\'d1\',1.1,2.2)"); + } + + TestUtils.restartDataNodes(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use \"test\""); + + try (ResultSet resultSet = statement.executeQuery("select * from turbine1 order by time")) { + assertNotNull(resultSet); + int cnt = 0; + while (resultSet.next()) { + assertEquals(1, resultSet.getLong("time")); + assertEquals("1.1", resultSet.getString("s1")); + assertEquals("2.2", resultSet.getString("s2")); + cnt++; + } + assertEquals(1, cnt); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetConfigurationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetConfigurationTableIT.java new file mode 100644 index 0000000000000..8a8e0e5e08d54 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetConfigurationTableIT.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.Statement; +import java.util.Arrays; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSetConfigurationTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testSetConfigurationWithUndefinedConfigKey() { + String expectedExceptionMsg = + "301: ignored config items: [a] because they are immutable or undefined."; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + executeAndExpectException(statement, "set configuration \"a\"='false'", expectedExceptionMsg); + int configNodeNum = EnvFactory.getEnv().getConfigNodeWrapperList().size(); + int dataNodeNum = EnvFactory.getEnv().getDataNodeWrapperList().size(); + + for (int i = 0; i < configNodeNum; i++) { + executeAndExpectException( + statement, "set configuration a=\'false\' on " + i, expectedExceptionMsg); + } + for (int i = 0; i < dataNodeNum; i++) { + int dnId = configNodeNum + i; + executeAndExpectException( + statement, "set configuration \"a\"='false' on " + dnId, expectedExceptionMsg); + } + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + private void executeAndExpectException( + Statement statement, String sql, String expectedContentInExceptionMsg) { + try { + statement.execute(sql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(expectedContentInExceptionMsg)); + return; + } + Assert.fail(); + } + + @Test + public void testSetConfiguration() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("set configuration \"enable_seq_space_compaction\"='false'"); + int configNodeNum = EnvFactory.getEnv().getConfigNodeWrapperList().size(); + int dataNodeNum = EnvFactory.getEnv().getDataNodeWrapperList().size(); + + for (int i = 0; i < configNodeNum; i++) { + statement.execute("set configuration enable_unseq_space_compaction=\'false\' on " + i); + } + for (int i = 0; i < dataNodeNum; i++) { + int dnId = configNodeNum + i; + statement.execute("set configuration \"enable_cross_space_compaction\"='false' on " + dnId); + statement.execute( + "set configuration inner_compaction_candidate_file_num='1',max_cross_compaction_candidate_file_num='1' on " + + dnId); + } + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + Assert.assertTrue( + EnvFactory.getEnv().getConfigNodeWrapperList().stream() + .allMatch( + nodeWrapper -> + checkConfigFileContains( + nodeWrapper, + "enable_seq_space_compaction=false", + "enable_unseq_space_compaction=false"))); + Assert.assertTrue( + EnvFactory.getEnv().getDataNodeWrapperList().stream() + .allMatch( + nodeWrapper -> + checkConfigFileContains( + nodeWrapper, + "enable_seq_space_compaction=false", + "enable_cross_space_compaction=false", + "inner_compaction_candidate_file_num=1", + "max_cross_compaction_candidate_file_num=1"))); + } + + private static boolean checkConfigFileContains( + AbstractNodeWrapper nodeWrapper, String... contents) { + try { + String systemPropertiesPath = + nodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String fileContent = new String(Files.readAllBytes(f.toPath())); + return Arrays.stream(contents).allMatch(fileContent::contains); + } catch (IOException ignore) { + return false; + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java new file mode 100644 index 0000000000000..5a3c435fb8b41 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.itbase.exception.InconsistentDataException; + +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSetSystemStatusTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void setSystemStatus() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("SET SYSTEM TO READONLY ON CLUSTER"); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollDelay(1, TimeUnit.SECONDS) + .until( + () -> { + ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); + int num = 0; + try { + while (resultSet.next()) { + String status = resultSet.getString("Status"); + if (status.equals("ReadOnly")) { + num++; + } + } + } catch (InconsistentDataException e) { + return false; + } + return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); + }); + + statement.execute("SET SYSTEM TO RUNNING ON CLUSTER"); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollDelay(1, TimeUnit.SECONDS) + .until( + () -> { + ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); + int num = 0; + try { + while (resultSet.next()) { + String status = resultSet.getString("Status"); + if (status.equals("Running")) { + num++; + } + } + } catch (InconsistentDataException e) { + return false; + } + return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); + }); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java new file mode 100644 index 0000000000000..824d93c5fbee7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java @@ -0,0 +1,408 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBWindowTVFIT { + private static final String DATABASE_NAME = "test"; + private static final String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table bid (stock_id string tag, price double field, s1 double field)", + "insert into bid values (2021-01-01T09:05:00, 'AAPL', 100.0, 101)", + "insert into bid values (2021-01-01T09:07:00, 'AAPL', 103.0, 101)", + "insert into bid values (2021-01-01T09:09:00, 'AAPL', 102.0, 101)", + "insert into bid values (2021-01-01T09:06:00, 'TESL', 200.0, 102)", + "insert into bid values (2021-01-01T09:07:00, 'TESL', 202.0, 202)", + "insert into bid values (2021-01-01T09:15:00, 'TESL', 195.0, 332)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + fail("insertData failed."); + } + } + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testHopFunction() { + String[] expectedHeader = + new String[] {"window_start", "window_end", "time", "stock_id", "price", "s1"}; + String[] retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:10:00.000Z,2021-01-01T09:20:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:25:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 5m, SIZE => 10m) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,AAPL,305.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,AAPL,305.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,TESL,402.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:15:00.000Z,TESL,402.0,", + "2021-01-01T09:10:00.000Z,2021-01-01T09:20:00.000Z,TESL,195.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:25:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 5m, SIZE => 10m) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T10:00:00.000Z,AAPL,305.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T10:00:00.000Z,TESL,597.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 1h, SIZE => 1h) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:07:00.000Z,2021-01-01T09:08:00.000Z,AAPL,103.0,", + "2021-01-01T09:09:00.000Z,2021-01-01T09:10:00.000Z,AAPL,102.0,", + "2021-01-01T09:07:00.000Z,2021-01-01T09:08:00.000Z,TESL,202.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:16:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 1m, SIZE => 1m, ORIGIN => 2021-01-01T09:07:00) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = + new String[] { + "2021-01-01T09:07:00.000Z,2021-01-01T09:08:00.000Z,AAPL,103.0,", + "2021-01-01T09:07:00.000Z,2021-01-01T09:08:00.000Z,TESL,202.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 1h, SIZE => 1m, ORIGIN => 2021-01-01T09:07:00) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => -300000, SIZE => 600000) ORDER BY stock_id, time", + "Invalid scalar argument SLIDE, should be a positive value", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 300000, SIZE => -600000) ORDER BY stock_id, time", + "Invalid scalar argument SIZE, should be a positive value", + DATABASE_NAME); + } + + @Test + public void testSessionFunction() { + String[] expectedHeader = + new String[] {"window_start", "window_end", "time", "stock_id", "price", "s1"}; + String[] retArray = + new String[] { + "2021-01-01T09:05:00.000Z,2021-01-01T09:09:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:09:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:05:00.000Z,2021-01-01T09:09:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:07:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:07:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:15:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM SESSION(DATA => bid PARTITION BY stock_id ORDER BY time, TIMECOL => 'time', GAP => 2m) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:05:00.000Z,2021-01-01T09:09:00.000Z,AAPL,305.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:07:00.000Z,TESL,402.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM SESSION(DATA => bid PARTITION BY stock_id ORDER BY time, GAP => 2m) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testVariationFunction() { + String[] expectedHeader = new String[] {"window_index", "time", "stock_id", "price", "s1"}; + String[] retArray = + new String[] { + "0,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "1,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "1,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "0,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "0,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "1,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM VARIATION(DATA => bid PARTITION BY stock_id ORDER BY time, COL => 'price', DELTA => 2.0) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + expectedHeader = new String[] {"start_time", "end_time", "stock_id", "avg"}; + retArray = + new String[] { + "2021-01-01T09:05:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,", + "2021-01-01T09:07:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.5,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:07:00.000Z,TESL,201.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT first(time) as start_time, last(time) as end_time, stock_id, avg(price) as avg FROM VARIATION(DATA => bid PARTITION BY stock_id ORDER BY time, COL => 'price', DELTA => 2.0) GROUP BY window_index, stock_id ORDER BY stock_id, window_index", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testCapacityFunction() { + String[] expectedHeader = new String[] {"window_index", "time", "stock_id", "price", "s1"}; + String[] retArray = + new String[] { + "0,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "0,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "1,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "0,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "0,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "1,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM CAPACITY(DATA => bid PARTITION BY stock_id ORDER BY time, SIZE => 2) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + expectedHeader = new String[] {"start_time", "end_time", "stock_id", "avg"}; + retArray = + new String[] { + "2021-01-01T09:05:00.000Z,2021-01-01T09:07:00.000Z,AAPL,101.5,", + "2021-01-01T09:09:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:07:00.000Z,TESL,201.0,", + "2021-01-01T09:15:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT first(time) as start_time, last(time) as end_time, stock_id, avg(price) as avg FROM CAPACITY(DATA => bid PARTITION BY stock_id ORDER BY time, SIZE => 2) GROUP BY window_index, stock_id ORDER BY stock_id, window_index", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testTumbleFunction() { + // TUMBLE (10m) + String[] expectedHeader = + new String[] {"window_start", "window_end", "time", "stock_id", "price", "s1"}; + String[] retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:10:00.000Z,2021-01-01T09:20:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM TUMBLE(DATA => bid, TIMECOL => 'time', SIZE => 10m) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + + // TUMBLE (10m) + origin + expectedHeader = new String[] {"window_start", "window_end", "time", "stock_id", "price", "s1"}; + retArray = + new String[] { + "2021-01-01T09:08:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:08:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM TUMBLE(DATA => bid, TIMECOL => 'time', SIZE => 10m, ORIGIN => 2021-01-01T09:08:00) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + + // TUMBLE (10m) + GROUP BY + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,AAPL,305.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:10:00.000Z,TESL,402.0,", + "2021-01-01T09:10:00.000Z,2021-01-01T09:20:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM TUMBLE(DATA => bid, TIMECOL => 'time', SIZE => 10m) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + + // TUMBLE (1h) + GROUP BY + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T10:00:00.000Z,AAPL,305.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T10:00:00.000Z,TESL,597.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM TUMBLE(DATA => bid, TIMECOL => 'time', SIZE => 1h) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM TUMBLE(DATA => bid, TIMECOL => 'time', SIZE => 0m) ORDER BY stock_id, time", + "Invalid scalar argument SIZE, should be a positive value", + DATABASE_NAME); + } + + @Test + public void testCumulateFunction() { + String[] expectedHeader = + new String[] {"window_start", "window_end", "time", "stock_id", "price", "s1"}; + String[] retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T09:06:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:05:00.000Z,AAPL,100.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:12:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + "2021-01-01T09:12:00.000Z,2021-01-01T09:24:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"window_start", "window_end", "time", "stock_id", "price", "s1"}; + retArray = + new String[] { + "2021-01-01T09:06:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:07:00.000Z,AAPL,103.0,101.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:09:00.000Z,AAPL,102.0,101.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:06:00.000Z,TESL,200.0,102.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:12:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:07:00.000Z,TESL,202.0,202.0,", + "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", + }; + tableResultSetEqualTest( + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m, ORIGIN => 2021-01-01T09:06:00) ORDER BY stock_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T09:06:00.000Z,AAPL,100.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,AAPL,305.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T09:12:00.000Z,TESL,402.0,", + "2021-01-01T09:12:00.000Z,2021-01-01T09:18:00.000Z,TESL,195.0,", + "2021-01-01T09:12:00.000Z,2021-01-01T09:24:00.000Z,TESL,195.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"window_start", "window_end", "stock_id", "sum"}; + retArray = + new String[] { + "2021-01-01T09:00:00.000Z,2021-01-01T10:00:00.000Z,AAPL,305.0,", + "2021-01-01T09:00:00.000Z,2021-01-01T10:00:00.000Z,TESL,597.0,", + }; + tableResultSetEqualTest( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 1h, SIZE => 1h) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + expectedHeader, + retArray, + DATABASE_NAME); + + // test UDFException + tableAssertTestFail( + "SELECT window_start, window_end, stock_id, sum(price) as sum FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 4m, SIZE => 10m) GROUP BY window_start, window_end, stock_id ORDER BY stock_id, window_start", + "Cumulative table function requires size must be an integral multiple of step.", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 0m, SIZE => 5m) ORDER BY stock_id, time", + "Invalid scalar argument STEP, should be a positive value", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 1m, SIZE => 0m) ORDER BY stock_id, time", + "Invalid scalar argument SIZE, should be a positive value", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBSQLFunctionManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBSQLFunctionManagementIT.java new file mode 100644 index 0000000000000..769567860e152 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBSQLFunctionManagementIT.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.db.it.udf; + +import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction; +import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction; +import org.apache.iotdb.db.queryengine.plan.relational.function.TableBuiltinTableFunction; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_USER_DEFINED_AGG_FUNC; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSQLFunctionManagementIT { + + private static final int BUILTIN_SCALAR_FUNCTIONS_COUNT = + TableBuiltinScalarFunction.getBuiltInScalarFunctionName().size(); + private static final int BUILTIN_AGGREGATE_FUNCTIONS_COUNT = + TableBuiltinAggregationFunction.values().length; + private static final int BUILTIN_TABLE_FUNCTIONS_COUNT = + TableBuiltinTableFunction.values().length; + private static final int BUILTIN_FUNCTIONS_COUNT = + BUILTIN_SCALAR_FUNCTIONS_COUNT + + BUILTIN_AGGREGATE_FUNCTIONS_COUNT + + BUILTIN_TABLE_FUNCTIONS_COUNT; + + private static final String UDF_LIB_PREFIX = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator; + + private static final String UDF_JAR_PREFIX = new File(UDF_LIB_PREFIX).toURI().toString(); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @After + public void dropAll() { + SQLFunctionUtils.dropAllUDF(); + } + + @Test + public void testCreateShowDropScalarFunction() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'"); + try (ResultSet resultSet = statement.executeQuery("show functions")) { + assertEquals(4, resultSet.getMetaData().getColumnCount()); + int count = 0; + while (resultSet.next()) { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); ++i) { + stringBuilder.append(resultSet.getString(i)).append(","); + } + String result = stringBuilder.toString(); + if (result.contains(FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC)) { + Assert.assertEquals( + String.format( + "UDSF,%s,org.apache.iotdb.db.query.udf.example.relational.ContainNull,AVAILABLE,", + FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC), + result); + } + ++count; + } + Assert.assertEquals(1 + BUILTIN_FUNCTIONS_COUNT, count); + } + statement.execute("drop function udsf"); + try (ResultSet resultSet = statement.executeQuery("show functions")) { + assertEquals(4, resultSet.getMetaData().getColumnCount()); + int count = 0; + while (resultSet.next()) { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); ++i) { + stringBuilder.append(resultSet.getString(i)).append(","); + } + String result = stringBuilder.toString(); + if (result.contains(FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC)) { + Assert.fail(); + } + ++count; + } + Assert.assertEquals(BUILTIN_FUNCTIONS_COUNT, count); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCreateShowDropAggregateFunction() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + SQLFunctionUtils.createUDF( + "udaf", "org.apache.iotdb.db.query.udf.example.relational.MyCount"); + + try (ResultSet resultSet = statement.executeQuery("show functions")) { + assertEquals(4, resultSet.getMetaData().getColumnCount()); + int count = 0; + while (resultSet.next()) { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); ++i) { + stringBuilder.append(resultSet.getString(i)).append(","); + } + String result = stringBuilder.toString(); + if (result.contains(FUNCTION_TYPE_USER_DEFINED_AGG_FUNC)) { + Assert.assertEquals( + String.format( + "UDAF,%s,org.apache.iotdb.db.query.udf.example.relational.MyCount,AVAILABLE,", + FUNCTION_TYPE_USER_DEFINED_AGG_FUNC), + result); + } + ++count; + } + Assert.assertEquals(1 + BUILTIN_FUNCTIONS_COUNT, count); + } + statement.execute("drop function udaf"); + try (ResultSet resultSet = statement.executeQuery("show functions")) { + assertEquals(4, resultSet.getMetaData().getColumnCount()); + int count = 0; + while (resultSet.next()) { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); ++i) { + stringBuilder.append(resultSet.getString(i)).append(","); + } + String result = stringBuilder.toString(); + if (result.contains(FUNCTION_TYPE_USER_DEFINED_AGG_FUNC)) { + Assert.fail(); + } + ++count; + } + Assert.assertEquals(BUILTIN_FUNCTIONS_COUNT, count); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCreateFunctionWithBuiltinFunctionName() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute( + "create function COS as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'"); + fail(); + } catch (SQLException throwable) { + assertTrue( + throwable + .getMessage() + .contains("the given function name conflicts with the built-in function name")); + } + try { + statement.execute( + "create function aVg as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'"); + fail(); + } catch (SQLException throwable) { + assertTrue( + throwable + .getMessage() + .contains("the given function name conflicts with the built-in function name")); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCreateFunctionTwice() throws SQLException { // create function twice + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'"); + try { + statement.execute( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'"); + fail(); + } catch (SQLException throwable) { + assertTrue(throwable.getMessage().contains("the same name UDF has been created")); + } + } + } + + @Test + public void testCreateInvalidFunction() throws SQLException { // create function twice + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull123'"); + fail(); + } catch (SQLException throwable) { + assertTrue(throwable.getMessage().contains("invalid")); + } + } + } + + @Test + public void testCreateFunctionWithURI() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull' using URI '%s'", + UDF_JAR_PREFIX + "udf-example.jar")); + statement.execute( + String.format( + "create function udsf2 as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull' using URI '%s'", + UDF_JAR_PREFIX + "udf-example.jar")); + + try (ResultSet resultSet = statement.executeQuery("show functions")) { + int count = 0; + while (resultSet.next()) { + ++count; + } + Assert.assertEquals(2 + BUILTIN_FUNCTIONS_COUNT, count); + assertEquals(4, resultSet.getMetaData().getColumnCount()); + } catch (Exception e) { + fail(); + } + } + } + + @Test + public void testCreateFunctionWithInvalidURI() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute( + String.format( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull' using URI '%s'", + "")); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().contains("701: Untrusted uri ")); + } + + try { + statement.execute( + String.format( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull' using URI '%s'", + "file:///data/udf/upload-test.jar")); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().contains("URI")); + } + } catch (SQLException throwable) { + fail(); + } + } + + @Test + public void testDropFunctionTwice() throws SQLException { // create + drop twice + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'"); + statement.execute("drop function udsf"); + + try { + // drop UDF that does not exist will not throw exception now. + statement.execute("drop function udsf"); + } catch (SQLException throwable) { + assertTrue(throwable.getMessage().contains("this UDF has not been created")); + } + } + } + + @Test + public void testDropNotExistFunction() { // drop + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + // drop UDF that does not exist will not throw exception now. + statement.execute("drop function udsf"); + } catch (SQLException throwable) { + assertTrue(throwable.getMessage().contains("this UDF has not been created")); + } + } + + @Test + public void testDropBuiltInFunction() throws SQLException { // drop + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute("drop function abs"); + fail(); + } catch (SQLException throwable) { + assertTrue( + throwable.getMessage().contains("Built-in function ABS can not be deregistered")); + } + // ensure that abs is not dropped + statement.execute("CREATE DATABASE db"); + statement.execute("USE db"); + statement.execute("CREATE TABLE table0 (device string TAG, s1 INT32)"); + statement.execute("INSERT INTO table0 (time, device, s1) VALUES (1, 'd1', -10)"); + try (ResultSet rs = statement.executeQuery("SELECT time, ABS(s1) FROM table0")) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(1, rs.getLong(1)); + Assert.assertEquals(10, rs.getInt(2)); + Assert.assertFalse(rs.next()); + } finally { + statement.execute("DROP DATABASE db"); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedAggregateFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedAggregateFunctionIT.java new file mode 100644 index 0000000000000..7ef7fd3b59033 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedAggregateFunctionIT.java @@ -0,0 +1,743 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it.udf; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUserDefinedAggregateFunctionIT { + private static final String DATABASE_NAME = "test"; + protected static final String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(province STRING TAG, city STRING TAG, region STRING TAG, device_id STRING TAG, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d05','red','A',30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','pudong','d05','red','A',35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d05','red','A',40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d05','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d05','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d09','red','A',30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','chaoyang','d09','red','A',35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d09','red','A',40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d09','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d09','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d13','red','A',30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','haidian','d13','red','A',35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d13','red','A',40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d13','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d13','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','haidian','d15','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d15','yellow','A',36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','haidian','d15','yellow','A',41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','haidian','d15','yellow','A',46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','haidian','d15','yellow','A',51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @After + public void dropFunction() { + SQLFunctionUtils.dropAllUDF(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + fail("insertData failed."); + } + } + + @Test + public void testMyCount() { + SQLFunctionUtils.createUDF( + "my_count", "org.apache.iotdb.db.query.udf.example.relational.MyCount"); + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select my_count(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, my_count(time) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, my_count(time) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,0,0,1,0,1,0,1,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, my_count(s1), my_count(s2), my_count(s3), my_count(s4), my_count(s5), my_count(s6), my_count(s7), my_count(s8), my_count(s9), my_count(s10) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,chaoyang,d10,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,chaoyang,d11,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,chaoyang,d12,1,1,1,1,1,1,1,1,3,1,", + "beijing,beijing,haidian,d13,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,haidian,d14,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,haidian,d15,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,haidian,d16,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,huangpu,d01,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,huangpu,d02,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,huangpu,d03,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,huangpu,d04,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,pudong,d05,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,pudong,d06,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,pudong,d07,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,pudong,d08,1,1,1,1,1,1,1,1,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, my_count(s1), my_count(s2), my_count(s3), my_count(s4), my_count(s5), my_count(s6), my_count(s7), my_count(s8), my_count(s9), my_count(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,5,", + "beijing,beijing,chaoyang,d10,3,", + "beijing,beijing,chaoyang,d11,5,", + "beijing,beijing,chaoyang,d12,3,", + "beijing,beijing,haidian,d13,5,", + "beijing,beijing,haidian,d14,3,", + "beijing,beijing,haidian,d15,5,", + "beijing,beijing,haidian,d16,3,", + "shanghai,shanghai,huangpu,d01,5,", + "shanghai,shanghai,huangpu,d02,3,", + "shanghai,shanghai,huangpu,d03,5,", + "shanghai,shanghai,huangpu,d04,3,", + "shanghai,shanghai,pudong,d05,5,", + "shanghai,shanghai,pudong,d06,3,", + "shanghai,shanghai,pudong,d07,5,", + "shanghai,shanghai,pudong,d08,3,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,my_count(time) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,16,", + "beijing,beijing,haidian,16,", + "shanghai,shanghai,huangpu,16,", + "shanghai,shanghai,pudong,16,", + }; + tableResultSetEqualTest( + "select province,city,region,my_count(time) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,32,", "shanghai,shanghai,32,", + }; + tableResultSetEqualTest( + "select province,city,my_count(time) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,32,", "shanghai,32,", + }; + tableResultSetEqualTest( + "select province,my_count(time) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest( + "select my_count(time) from table1", expectedHeader, retArray, DATABASE_NAME); + // drop function and invoke + dropFunction(); + tableAssertTestFail("select my_count(time) from table1", "Unknown function", DATABASE_NAME); + } + + @Test + public void testMyAvg() { + SQLFunctionUtils.createUDF("my_avg", "org.apache.iotdb.db.query.udf.example.relational.MyAvg"); + String[] expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + String[] retArray = + new String[] { + "d01,red,A,45.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, my_avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + retArray = + new String[] { + "d01,red,A,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, my_avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, my_avg(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, my_avg(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,42500.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,38500.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,42500.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,38500.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,42500.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,38500.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,42500.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,38500.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, my_avg(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,44.5,", + "beijing,beijing,haidian,44.5,", + "shanghai,shanghai,huangpu,44.5,", + "shanghai,shanghai,pudong,44.5,", + }; + tableResultSetEqualTest( + "select province,city,region,my_avg(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,44.5,", "shanghai,shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,city,my_avg(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,44.5,", "shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,my_avg(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"44.5,"}; + tableResultSetEqualTest( + "select my_avg(s4) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testIllegalInput() { + SQLFunctionUtils.createUDF( + "first_two_sum", "org.apache.iotdb.db.query.udf.example.relational.FirstTwoSum"); + tableAssertTestFail( + "select first_two_sum(s1,s2) from table1 group by device_id", + "FirstTwoSum should accept three column as input", + DATABASE_NAME); + tableAssertTestFail( + "select first_two_sum(s1,s2,s3) from table1 group by device_id", + "FirstTwoSum should accept TIMESTAMP as the third input", + DATABASE_NAME); + tableAssertTestFail( + "select first_two_sum(s1,s10,time) from table1 group by device_id", + "FirstTwoSum should accept INT32, INT64, FLOAT, DOUBLE as the first two inputs", + DATABASE_NAME); + } + + @Test + public void testFirstTwoSum() { + SQLFunctionUtils.createUDF( + "first_two_sum", "org.apache.iotdb.db.query.udf.example.relational.FirstTwoSum"); + String[] expectedHeader = new String[] {"device_id", "sum"}; + String[] retArray = + new String[] { + "d01,35030.0,", + "d02,50036.0,", + "d03,31036.0,", + "d04,40055.0,", + "d05,35030.0,", + "d06,50036.0,", + "d07,31036.0,", + "d08,40055.0,", + "d09,35030.0,", + "d10,50036.0,", + "d11,31036.0,", + "d12,40055.0,", + "d13,35030.0,", + "d14,50036.0,", + "d15,31036.0,", + "d16,40055.0,", + }; + + tableResultSetEqualTest( + "select device_id, first_two_sum(s1, s2, time) as sum from table1 group by device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select device_id, first_two_sum(s1, s2, s9) as sum from table1 group by device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDistinct() { + SQLFunctionUtils.createUDF( + "my_count", "org.apache.iotdb.db.query.udf.example.relational.MyCount"); + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "2,2,4,16,5,5,5,5,2,24,32,5,10,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select my_count(distinct province), my_count(distinct city), my_count(distinct region), my_count(distinct device_id), my_count(distinct s1), my_count(distinct s2), my_count(distinct s3), my_count(distinct s4), my_count(distinct s5), my_count(distinct s6), my_count(distinct s7), my_count(distinct s8), my_count(distinct s9), my_count(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,5,5,5,2,6,8,5,10,1,", + "beijing,beijing,haidian,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,huangpu,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,pudong,5,5,5,5,2,6,8,5,10,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, my_count(distinct s1), my_count(distinct s2), my_count(distinct s3), my_count(distinct s4), my_count(distinct s5), my_count(distinct s6), my_count(distinct s7), my_count(distinct s8), my_count(distinct s9), my_count(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedAggregationFunctionNonStreamIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedAggregationFunctionNonStreamIT.java new file mode 100644 index 0000000000000..c74e748732a54 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedAggregationFunctionNonStreamIT.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it.udf; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUserDefinedAggregationFunctionNonStreamIT + extends IoTDBUserDefinedAggregateFunctionIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + String original = sqls[2]; + // make 'province', 'city', 'region' be FIELD to cover cases using GroupedAccumulator + sqls[2] = + "CREATE TABLE table1(province STRING FIELD, city STRING FIELD, region STRING FIELD, device_id STRING TAG, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)"; + prepareTableData(sqls); + // rollback original content + sqls[2] = original; + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedScalarFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedScalarFunctionIT.java new file mode 100644 index 0000000000000..2d1b741a7c62b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedScalarFunctionIT.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it.udf; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Types; +import java.text.DecimalFormat; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUserDefinedScalarFunctionIT { + private static String[] sqls = + new String[] { + "CREATE DATABASE test", + "USE test", + "CREATE TABLE vehicle (device_id string tag, s1 INT32 field, s2 INT64 field, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD)", + "insert into vehicle(time, device_id, s1, s2, s3, s4, s5) values (1, 'd0', 1, 1, 1.1, 1.1, true)", + "insert into vehicle(time, device_id, s1, s2, s3, s4, s5) values (2, 'd0', null, 2, 2.2, 2.2, true)", + "insert into vehicle(time, device_id, s1, s2, s3, s4, s5) values (3, 'd0', 3, 3, null, null, false)", + "insert into vehicle(time, device_id, s5) values (5, 'd0', true)", + "CREATE FUNCTION contain_null as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", + "CREATE FUNCTION all_sum as 'org.apache.iotdb.db.query.udf.example.relational.AllSum'", + "CREATE TABLE t2 (device_id string tag, s1 DATE field)", + "insert into t2(time, device_id, s1) values (1, 'd0', '2024-02-28')", + "insert into t2(time, device_id, s1) values (2, 'd0', '2024-02-29')", + "insert into t2(time, device_id, s1) values (3, 'd0', '2024-03-01')", + "CREATE FUNCTION date_plus as 'org.apache.iotdb.db.query.udf.example.relational.DatePlus'" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + System.out.println(sql); + statement.execute(sql); + } + } catch (Exception e) { + fail("insertData failed."); + } + } + + @Test + public void testIllegalInput() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + try { + statement.execute("select contain_null() from vehicle"); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("At least one parameter is required")); + } + try { + statement.execute("select all_sum(s1,s2,s3,s4,s5) from vehicle"); + fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("Only support inputs of INT32,INT64,DOUBLE,FLOAT type")); + } + + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testNormalQuery() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + List expectedResult = + Arrays.asList("1,false,false", "2,true,true", "3,true,false", "5,true,true"); + int row = 0; + try (ResultSet resultSet = + statement.executeQuery( + "select time, contain_null(s1,s2,s3,s4,s5) as contain_null, contain_null(s1) as s1_null from vehicle")) { + while (resultSet.next()) { + Assert.assertEquals( + expectedResult.get(row), + resultSet.getLong(1) + "," + resultSet.getBoolean(2) + "," + resultSet.getBoolean(3)); + row++; + } + assertEquals(4, row); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testPolymorphicQuery() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + List expectedResult = + Arrays.asList( + "1,1,1,1.1,1.1,2,3.1,4.2,2.1", + "2,0,2,2.2,2.2,2,4.2,6.4,4.2", + "3,3,3,.0,.0,6,6.0,6.0,3.0", + "5,0,0,.0,.0,0,.0,.0,.0"); + int row = 0; + try (ResultSet resultSet = + statement.executeQuery( + "select time, all_sum(s1) as s1, all_sum(s2) as s2, all_sum(s3) as s3, all_sum(s4) as s4, all_sum(s1,s2) as s12, all_sum(s1,s2,s3) as s123, all_sum(s1,s2,s3,s4) as s1234, all_sum(s2,s3) as s23 from vehicle")) { + Assert.assertEquals(Types.TIMESTAMP, resultSet.getMetaData().getColumnType(1)); + Assert.assertEquals(Types.INTEGER, resultSet.getMetaData().getColumnType(2)); + Assert.assertEquals(Types.BIGINT, resultSet.getMetaData().getColumnType(3)); + Assert.assertEquals(Types.FLOAT, resultSet.getMetaData().getColumnType(4)); + Assert.assertEquals(Types.DOUBLE, resultSet.getMetaData().getColumnType(5)); + Assert.assertEquals(Types.BIGINT, resultSet.getMetaData().getColumnType(6)); + Assert.assertEquals(Types.FLOAT, resultSet.getMetaData().getColumnType(7)); + Assert.assertEquals(Types.DOUBLE, resultSet.getMetaData().getColumnType(8)); + Assert.assertEquals(Types.FLOAT, resultSet.getMetaData().getColumnType(9)); + DecimalFormat df = new DecimalFormat("#.0"); + while (resultSet.next()) { + Assert.assertEquals( + expectedResult.get(row), + resultSet.getLong(1) + + "," + + resultSet.getInt(2) + + "," + + resultSet.getLong(3) + + "," + + df.format(resultSet.getFloat(4)) + + "," + + df.format(resultSet.getDouble(5)) + + "," + + resultSet.getLong(6) + + "," + + df.format(resultSet.getFloat(7)) + + "," + + df.format(resultSet.getDouble(8)) + + "," + + df.format(resultSet.getFloat(9))); + row++; + } + assertEquals(4, row); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testDateFunction() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + List expectedResult = + Arrays.asList( + LocalDate.of(2024, 2, 29), LocalDate.of(2024, 3, 1), LocalDate.of(2024, 3, 2)); + int row = 0; + try (ResultSet resultSet = statement.executeQuery("select date_plus(s1, 1) from t2")) { + while (resultSet.next()) { + Assert.assertEquals(expectedResult.get(row), resultSet.getDate(1).toLocalDate()); + row++; + } + assertEquals(3, row); + } + expectedResult = + Arrays.asList( + LocalDate.of(2024, 3, 1), LocalDate.of(2024, 3, 2), LocalDate.of(2024, 3, 3)); + row = 0; + try (ResultSet resultSet = statement.executeQuery("select date_plus(s1, 2) from t2")) { + while (resultSet.next()) { + Assert.assertEquals(expectedResult.get(row), resultSet.getDate(1).toLocalDate()); + row++; + } + assertEquals(3, row); + } + + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testUntrustedUri() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "CREATE FUNCTION test as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'"); + fail("should fail"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("701: Untrusted uri ")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java new file mode 100644 index 0000000000000..7fe6648fe70ba --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it.udf; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUserDefinedTableFunctionIT { + private static final String DATABASE_NAME = "test"; + private static final String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE vehicle (device_id string tag, s1 INT32 field, s2 INT64 field)", + "insert into vehicle(time, device_id, s1, s2) values (1, 'd0', 1, 1)", + "insert into vehicle(time, device_id, s1, s2) values (2, 'd0', null, 2)", + "insert into vehicle(time, device_id, s1, s2) values (3, 'd0', 3, 3)", + "insert into vehicle(time, device_id, s1) values (5, 'd1', 4)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @After + public void dropFunction() { + SQLFunctionUtils.dropAllUDF(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + System.out.println(sql + ";"); + statement.execute(sql); + } + } catch (Exception e) { + fail("insertData failed."); + } + } + + @Test + public void testMySplit() { + // split: table function without table parameter + SQLFunctionUtils.createUDF("split", "org.apache.iotdb.db.query.udf.example.relational.MySplit"); + String[] expectedHeader = new String[] {"output"}; + String[] retArray = + new String[] { + "1,", "2,", "3,", "4,", "5,", + }; + tableResultSetEqualTest( + "select * from split('1,2,3,4,5')", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "select * from table(split('1,2,3,4,5'))", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "select * from split('1+2+3+4+5','\\+')", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "select * from table(split('1+2+3+4+5','\\+'))", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "select * from split(INPUT=>'1-2-3-4-5',split=>'-')", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select * from table(split(INPUT=>'1-2-3-4-5',split=>'-'))", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"o1", "o2"}; + retArray = + new String[] { + "2,2,", "4,4,", + }; + tableResultSetEqualTest( + "select * from TABLE(SPLIT('1,2,4,5')) a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select * from SPLIT('1,2,4,5') a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testMyRepeat() { + // repeat1: row semantic, pass through, with proper column + // repeat2: row semantic, pass through, without proper column + SQLFunctionUtils.createUDF( + "repeat1", "org.apache.iotdb.db.query.udf.example.relational.MyRepeatWithIndex"); + SQLFunctionUtils.createUDF( + "repeat2", "org.apache.iotdb.db.query.udf.example.relational.MyRepeatWithoutIndex"); + String[] expectedHeader = new String[] {"repeat_index", "time", "device_id", "s1", "s2"}; + String[] retArray = + new String[] { + "0,1970-01-01T00:00:00.001Z,d0,1,1,", + "1,1970-01-01T00:00:00.001Z,d0,1,1,", + "0,1970-01-01T00:00:00.002Z,d0,null,2,", + "1,1970-01-01T00:00:00.002Z,d0,null,2,", + "0,1970-01-01T00:00:00.003Z,d0,3,3,", + "1,1970-01-01T00:00:00.003Z,d0,3,3,", + "0,1970-01-01T00:00:00.005Z,d1,4,null,", + "1,1970-01-01T00:00:00.005Z,d1,4,null,", + }; + tableResultSetEqualTest( + "select * from TABLE(repeat1(TABLE(vehicle), 2)) order by time,repeat_index", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select * from repeat1(vehicle, 2) order by time,repeat_index", + expectedHeader, + retArray, + DATABASE_NAME); + expectedHeader = new String[] {"time", "device_id", "s1"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,1,", + "1970-01-01T00:00:00.001Z,d0,1,", + "1970-01-01T00:00:00.002Z,d0,null,", + "1970-01-01T00:00:00.002Z,d0,null,", + "1970-01-01T00:00:00.003Z,d0,3,", + "1970-01-01T00:00:00.003Z,d0,3,", + "1970-01-01T00:00:00.005Z,d1,4,", + "1970-01-01T00:00:00.005Z,d1,4,", + }; + tableResultSetEqualTest( + "select * from TABLE(repeat2(TABLE(select time, device_id, s1 from vehicle), 2)) order by time", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select * from repeat2((select time, device_id, s1 from vehicle), 2) order by time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testHybrid() { + SQLFunctionUtils.createUDF( + "repeat", "org.apache.iotdb.db.query.udf.example.relational.MyRepeatWithoutIndex"); + SQLFunctionUtils.createUDF( + "exclude", "org.apache.iotdb.db.query.udf.example.relational.MyExcludeColumn"); + SQLFunctionUtils.createUDF("split", "org.apache.iotdb.db.query.udf.example.relational.MySplit"); + String[] expectedHeader = new String[] {"time", "device_id", "s1", "output"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,1,1,", + "1970-01-01T00:00:00.001Z,d0,1,1,", + "1970-01-01T00:00:00.001Z,d0,1,1,", + "1970-01-01T00:00:00.005Z,d1,4,4,", + "1970-01-01T00:00:00.005Z,d1,4,4,", + "1970-01-01T00:00:00.005Z,d1,4,4,", + }; + tableResultSetEqualTest( + "select * from " + + "TABLE(REPEAT(TABLE(select * from TABLE(EXCLUDE(TABLE(vehicle), 's2'))), 3)) a " + + "JOIN " + + "TABLE(SPLIT('1,4,6,8,10')) b " + + "ON a.s1=CAST(b.output AS INT32)" + + "ORDER BY time", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select * from " + + "REPEAT((select * from EXCLUDE(vehicle, 's2')), 3) a " + + "JOIN " + + "SPLIT('1,4,6,8,10') b " + + "ON a.s1=CAST(b.output AS INT32)" + + "ORDER BY time", + expectedHeader, + retArray, + DATABASE_NAME); + expectedHeader = new String[] {"device_id", "sum"}; + retArray = + new String[] { + "d0,9.0,", "d1,12.0,", + }; + tableResultSetEqualTest( + "select device_id, sum(s1) as sum from " + + "TABLE(REPEAT(TABLE(select * from vehicle where time>1), 3)) " + + "GROUP BY device_id " + + "ORDER BY device_id", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select device_id, sum(s1) as sum from " + + "REPEAT((select * from vehicle where time>1), 3) " + + "GROUP BY device_id " + + "ORDER BY device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testPassThroughPartitionColumn() { + SQLFunctionUtils.createUDF( + "MY_SELECT", "org.apache.iotdb.db.query.udf.example.relational.MySelectColumn"); + String[] expectedHeader = new String[] {"s1", "device_id"}; + String[] retArray = + new String[] { + "1,d0,", "null,d0,", "3,d0,", "4,d1,", + }; + tableResultSetEqualTest( + "select * from MY_SELECT(vehicle PARTITION BY device_id, 's1') ORDER BY device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testIllegalInput() { + SQLFunctionUtils.createUDF( + "repeat", "org.apache.iotdb.db.query.udf.example.relational.MyRepeatWithoutIndex"); + SQLFunctionUtils.createUDF( + "error", "org.apache.iotdb.db.query.udf.example.relational.MyErrorTableFunction"); + tableAssertTestFail( + "SELECT * FROM repeat(N=>2)", "Missing required argument: DATA", DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(vehicle, N=>2)", + "All arguments must be passed by name or all must be passed positionally", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(2, vehicle)", + "Invalid argument DATA. Expected table argument, got expression", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(DATA=>vehicle, N=>vehicle)", + "Invalid argument N. Expected scalar argument, got table", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(DATA=>vehicle, N=>1, N=>2)", + "Too many arguments. Expected at most 2 arguments, got 3 arguments", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(DATA=>vehicle, DATA=>vehicle)", + "Duplicate argument name: DATA", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(DATA=>vehicle, NUM=>3)", + "Unexpected argument name: NUM", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(DATA=>vehicle PARTITION BY device_id, NUM=>3)", + "Partitioning can not be specified for table argument with row semantics", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM repeat(DATA=>vehicle ORDER BY time, NUM=>3)", + "Ordering can not be specified for table argument with row semantics", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM error(vehicle, 0)", + "does not specify required input columns from table argument", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM error(vehicle, 1)", + "specifies empty list of required columns from table argument DATA", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM error(vehicle, 2)", + "specifies negative index of required column from table argument DATA", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM error(vehicle, 3)", "out of bounds for table with 4 columns", DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM error(vehicle, 4)", + "specifies required columns from table argument TIMECHO which cannot be found", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM not_register(vehicle, 4)", "Unknown function: not_register", DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/SQLFunctionUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/SQLFunctionUtils.java new file mode 100644 index 0000000000000..34f57b2940b6f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/SQLFunctionUtils.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it.udf; + +import org.apache.iotdb.it.env.EnvFactory; + +import org.junit.Assert; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_USER_DEFINED_AGG_FUNC; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_USER_DEFINED_TABLE_FUNC; +import static org.junit.Assert.fail; + +public class SQLFunctionUtils { + public static void createUDF(String udfName, String classPath) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + // create + statement.execute(String.format("create function %s as '%s'", udfName, classPath)); + // check + try (ResultSet resultSet = statement.executeQuery("show functions")) { + boolean found = false; + while (resultSet.next()) { + if (resultSet.getString(1).equals(udfName.toUpperCase()) + && resultSet.getString(3).equals(classPath)) { + found = true; + break; + } + } + Assert.assertTrue("Can not find function", found); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + + public static void dropAllUDF() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + List udfName = new ArrayList<>(); + try (ResultSet resultSet = statement.executeQuery("show functions")) { + Set externalUDF = + new HashSet<>( + Arrays.asList( + FUNCTION_TYPE_USER_DEFINED_SCALAR_FUNC, + FUNCTION_TYPE_USER_DEFINED_AGG_FUNC, + FUNCTION_TYPE_USER_DEFINED_TABLE_FUNC)); + while (resultSet.next()) { + if (externalUDF.contains(resultSet.getString(2))) { + udfName.add(resultSet.getString(1)); + } + } + } + for (String name : udfName) { + statement.execute(String.format("drop function %s", name)); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java new file mode 100644 index 0000000000000..118554d8aadf4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.relational.it.mqtt; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.apache.tsfile.read.common.Field; +import org.fusesource.mqtt.client.BlockingConnection; +import org.fusesource.mqtt.client.MQTT; +import org.fusesource.mqtt.client.QoS; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBMQTTServiceIT { + private BlockingConnection connection; + private static final String IP = System.getProperty("RemoteIp", "127.0.0.1"); + private static final String USER = System.getProperty("RemoteUser", "root"); + private static final String PASSWORD = System.getProperty("RemotePassword", "root"); + private static final String DATABASE = "mqtttest"; + public static final String FORMATTER = "line"; + + @Before + public void setUp() throws Exception { + BaseEnv baseEnv = EnvFactory.getEnv(); + baseEnv.getConfig().getDataNodeConfig().setEnableMQTTService(true); + baseEnv.getConfig().getDataNodeConfig().setMqttPayloadFormatter(FORMATTER); + baseEnv.initClusterEnvironment(); + DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + int port = portConflictDataNodeWrapper.getMqttPort(); + MQTT mqtt = new MQTT(); + mqtt.setHost(IP, port); + mqtt.setUserName(USER); + mqtt.setPassword(PASSWORD); + mqtt.setConnectAttemptsMax(3); + mqtt.setReconnectDelay(10); + + connection = mqtt.blockingConnection(); + connection.connect(); + } + + @After + public void tearDown() throws Exception { + try { + if (connection != null) { + connection.disconnect(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testNoAttr() throws Exception { + try (final ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE)) { + session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + String payload1 = "test1,tag1=t1,tag2=t2 field1=1,field2=1f,field3=1i32 1"; + connection.publish(DATABASE + "/myTopic", payload1.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(1000); + try (final SessionDataSet dataSet = + session.executeQueryStatement( + "select tag1,tag2,field1,field2,field3 from test1 where time = 1")) { + assertEquals(5, dataSet.getColumnNames().size()); + List fields = dataSet.next().getFields(); + assertEquals("t1", fields.get(0).getStringValue()); + assertEquals("t2", fields.get(1).getStringValue()); + assertEquals(1d, fields.get(2).getDoubleV(), 0); + assertEquals(1f, fields.get(3).getFloatV(), 0); + assertEquals(1, fields.get(4).getIntV(), 0); + } + } + } + + @Test + public void testWithAttr() throws Exception { + try (final ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE)) { + session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + String payload1 = "test2,tag1=t1,tag2=t2 attr3=a3,attr4=a4 field1=1,field2=1f,field3=1i32 1"; + connection.publish(DATABASE + "/myTopic", payload1.getBytes(), QoS.AT_LEAST_ONCE, false); + Thread.sleep(1000); + try (final SessionDataSet dataSet = + session.executeQueryStatement( + "select tag1,tag2,attr3,attr4,field1,field2,field3 from test2 where time = 1")) { + assertEquals(7, dataSet.getColumnNames().size()); + List fields = dataSet.next().getFields(); + assertEquals("t1", fields.get(0).getStringValue()); + assertEquals("t2", fields.get(1).getStringValue()); + assertEquals("a3", fields.get(2).getStringValue()); + assertEquals("a4", fields.get(3).getStringValue()); + assertEquals(1d, fields.get(4).getDoubleV(), 0); + assertEquals(1f, fields.get(5).getFloatV(), 0); + assertEquals(1, fields.get(6).getIntV(), 0); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java new file mode 100644 index 0000000000000..08e9ccc8635ad --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDatetimeFormatTableIT { + + private static final String DATABASE_NAME = "db"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false); + EnvFactory.getEnv().initClusterEnvironment(); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("create database " + DATABASE_NAME); + statement.execute("use " + DATABASE_NAME); + statement.execute( + "create table table1(device_id STRING TAG, s1 INT32 FIELD, s2 DOUBLE FIELD)"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDatetimeInputFormat() { + String[] datetimeStrings = { + "2022-01-01 01:02:03", // yyyy-MM-dd HH:mm:ss + "2022/01/02 01:02:03", // yyyy/MM/dd HH:mm:ss + "2022.01.03 01:02:03", // yyyy.MM.dd HH:mm:ss + "2022-01-04 01:02:03+01:00", // yyyy-MM-dd HH:mm:ssZZ + "2022/01/05 01:02:03+01:00", // yyyy/MM/dd HH:mm:ssZZ + "2022.01.06 01:02:03+01:00", // yyyy.MM.dd HH:mm:ssZZ + "2022-01-07 01:02:03.400", // yyyy-MM-dd HH:mm:ss.SSS + "2022/01/08 01:02:03.400", // yyyy/MM/dd HH:mm:ss.SSS + "2022.01.09 01:02:03.400", // yyyy.MM.dd HH:mm:ss.SSS + "2022-01-10 01:02:03.400+01:00", // yyyy-MM-dd HH:mm:ss.SSSZZ + "2022-01-11 01:02:03.400+01:00", // yyyy/MM/dd HH:mm:ss.SSSZZ + "2022-01-12 01:02:03.400+01:00", // yyyy.MM.dd HH:mm:ss.SSSZZ + "2022-01-13T01:02:03.400+01:00" // ISO8601 standard time format + }; + long[] timestamps = { + 1640970123000L, + 1641056523000L, + 1641142923000L, + 1641254523000L, + 1641340923000L, + 1641427323000L, + 1641488523400L, + 1641574923400L, + 1641661323400L, + 1641772923400L, + 1641859323400L, + 1641945723400L, + 1642032123400L + }; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + connection.setClientInfo("time_zone", "+08:00"); + statement.execute("use " + DATABASE_NAME); + + for (int i = 0; i < datetimeStrings.length; i++) { + String insertSql = + String.format( + "INSERT INTO table1(time, device_id, s1) values (%s, %s, %d)", + datetimeStrings[i], "'d1'", i); + statement.execute(insertSql); + } + + try (ResultSet resultSet = + statement.executeQuery("SELECT time, s1 FROM table1 where device_id='d1'")) { + Assert.assertNotNull(resultSet); + int cnt = 0; + while (resultSet.next()) { + Assert.assertEquals(timestamps[cnt], resultSet.getLong(1)); + cnt++; + } + Assert.assertEquals(timestamps.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void testBigDateTime() { + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + + statement.execute( + "insert into table1(time,device_id,s2) values (1618283005586000, 'd2',8.76)"); + + try (ResultSet resultSet = + statement.executeQuery( + "select time, s2 from table1 where device_id='d2' and time=53251-05-07T17:06:26.000+08:00")) { + Assert.assertNotNull(resultSet); + int cnt = 0; + while (resultSet.next()) { + Assert.assertEquals(1618283005586000L, resultSet.getLong(1)); + cnt++; + } + Assert.assertEquals(1, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterBetweenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterBetweenTableIT.java new file mode 100644 index 0000000000000..52d4fae12f8ba --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterBetweenTableIT.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFilterBetweenTableIT { + protected static final int ITERATION_TIMES = 10; + private static final String DATABASE_NAME = "test"; + private static final List SQLs = new ArrayList<>(); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + generateData(SQLs); + prepareTableData(SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void generateData(List SQLs) { + SQLs.add("CREATE DATABASE " + DATABASE_NAME); + SQLs.add("USE " + DATABASE_NAME); + SQLs.add( + "CREATE TABLE table1 (device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 TEXT FIELD)"); + for (int i = 1; i <= ITERATION_TIMES; ++i) { + SQLs.add( + String.format( + "insert into table1(device,time,s1,s2,s3) values('d1',%d,%d,%d,%s)", i, i, i, i)); + } + } + + @Test + public void testBetweenExpression() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + int start = 1, end = 5; + String query = "SELECT * FROM table1 WHERE s1 BETWEEN " + start + " AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM table1 WHERE s1 NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = "SELECT * FROM table1 WHERE time BETWEEN " + start + " AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM table1 WHERE time NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = "SELECT * FROM table1 WHERE " + start + " BETWEEN time AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(1), rs.getString("time")); + Assert.assertEquals("1", rs.getString("s1")); + Assert.assertEquals("1", rs.getString("s2")); + Assert.assertEquals("1", rs.getString("s3")); + } + + query = "SELECT * FROM table1 WHERE " + start + " NOT BETWEEN time AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start + 1; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = "SELECT * FROM table1 WHERE " + start + " BETWEEN " + end + " AND time"; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertFalse(rs.next()); + } + + query = "SELECT * FROM table1 WHERE " + start + " NOT BETWEEN " + end + " AND time"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = new String[] {"true,true,", "true,false,", "false,false,"}; + tableResultSetEqualTest( + "select s1 between 1 and 2, time between 0 and 1 from table1 where time between 1 and 3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"r1", "r2"}; + retArray = new String[] {"true,true,", "true,false,", "false,false,"}; + tableResultSetEqualTest( + "select s1 between 1 and 2 as r1, time between 0 and 1 as r2 from table1 where time between 1 and 3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"true,true,", "true,false,", "false,false,"}; + tableResultSetEqualTest( + "select s1 between 1 and 2, time between 0 and 1 from table1 where time between 1 and 3 order by device", + expectedHeader, + retArray, + DATABASE_NAME); + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterNullTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterNullTableIT.java new file mode 100644 index 0000000000000..a86e683fa49a6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterNullTableIT.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFilterNullTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE testNullFilter(device STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD, s3 DOUBLE FIELD)", + "INSERT INTO testNullFilter(time,device,s2,s3) " + "values(1, 'd1', false, 11.1)", + "INSERT INTO testNullFilter(time,device,s1,s2) " + "values(2, 'd1', 22, true)", + "INSERT INTO testNullFilter(time,device,s1,s3) " + "values(3, 'd1', 23, 33.3)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullFilterTest() { + String[] retArray = + new String[] { + defaultFormatDataTime(1) + ",null,false,11.1", + defaultFormatDataTime(2) + ",22,true,null", + defaultFormatDataTime(3) + ",23,null,33.3" + }; + try (Connection connectionIsNull = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connectionIsNull.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + int count = 0; + + ResultSet resultSet = statement.executeQuery("select * from testNullFilter where s1 is null"); + while (resultSet.next()) { + String ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(retArray[count], ans); + count++; + } + + resultSet = statement.executeQuery("select * from testNullFilter where s1 is not null"); + while (resultSet.next()) { + String ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(retArray[count], ans); + count++; + } + assertEquals(retArray.length, count); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterTableIT.java new file mode 100644 index 0000000000000..7ecc9673b2344 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBFilterTableIT.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFilterTableIT { + protected static final int ITERATION_TIMES = 10; + private static final String DATABASE_NAME = "test"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setUdfMemoryBudgetInMB(5); + EnvFactory.getEnv().initClusterEnvironment(); + createTimeSeries(); + generateData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void createTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE testNaN(device STRING TAG, n1 DOUBLE FIELD, n2 DOUBLE FIELD)"); + statement.execute( + "CREATE TABLE testTimeSeries(device STRING TAG, s1 BOOLEAN FIELD, s2 BOOLEAN FIELD)"); + statement.execute("CREATE TABLE testUDTF(device STRING TAG, s1 TEXT FIELD, s2 DOUBLE FIELD)"); + statement.execute("CREATE TABLE sg1(device STRING TAG, s1 DOUBLE FIELD, s2 TEXT FIELD)"); + + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void generateData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + for (int i = 0; i < ITERATION_TIMES; i++) { + statement.execute( + String.format("insert into testNaN(time,device,n1,n2) values(%d,'d1',%d,%d)", i, i, i)); + + switch (i % 3) { + case 0: + statement.execute( + String.format( + "insert into testTimeSeries(time,device,s1,s2) values(%d,'d1',true,true)", i)); + break; + case 1: + statement.execute( + String.format( + "insert into testTimeSeries(time,device,s1,s2) values(%d,'d1',true,false)", i)); + break; + case 2: + statement.execute( + String.format( + "insert into testTimeSeries(time,device,s1,s2) values(%d,'d1',false,false)", + i)); + break; + } + } + statement.execute(" insert into sg1(time, device, s1, s2) values (1,'d1',1,'1')"); + statement.execute(" insert into sg1(time, device, s1, s2) values (2,'d1',2,'2')"); + statement.execute(" insert into testUDTF(time, device, s1, s2) values (1,'d1','ss',0)"); + statement.execute(" insert into testUDTF(time, device, s1, s2) values (2,'d1','d',3)"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testFilterBooleanSeries() { + String[] expectedHeader = new String[] {"s1", "s2"}; + String[] retArray = new String[] {"true,true,", "true,true,", "true,true,", "true,true,"}; + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s2", expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s1 and s2", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "true,true,", + "true,false,", + "true,true,", + "true,false,", + "true,true,", + "true,false,", + "true,true," + }; + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s1 or s2", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s1 or false", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testFilterNaN() { + String sqlStr = "select n1 from testNaN where n1/n2 > 0"; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sqlStr); + int count = 0; + while (resultSet.next()) { + ++count; + } + + // 0.0/0.0 is NaN which should not be kept. + assertEquals(ITERATION_TIMES - 1, count); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testSameConstantWithDifferentType() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select s2 from sg1 where s1 = 1 and s2 >= '1' and s2 <= '2'"); + int count = 0; + while (resultSet.next()) { + ++count; + } + assertEquals(1, count); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testMismatchedDataTypes() { + tableAssertTestFail( + "select s1 from sg1 where s1", + "701: WHERE clause must evaluate to a boolean: actual type DOUBLE", + DATABASE_NAME); + // TODO After Aggregation supported + /*assertTestFail( + "select count(s1) from root.sg1.d1 group by ([0, 40), 5ms) having count(s1) + 1;", + "The output type of the expression in HAVING clause should be BOOLEAN, actual data type: DOUBLE."); + assertTestFail( + "select count(s1) from root.sg1.d1 group by ([0, 40), 5ms) having count(s1) + 1 align by device;", + "The output type of the expression in HAVING clause should be BOOLEAN, actual data type: DOUBLE.");*/ + } + + @Ignore // TODO After UDTF supported + @Test + public void testFilterWithUDTF() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet containsResultSet = + statement.executeQuery("select s1 from testUDTF where STRING_CONTAINS(s1, 's'='s')"); + ResultSet sinResultSet = + statement.executeQuery("select s1 from testUDTF where sin(s2) = 0")) { + int containsCnt = 0; + while (containsResultSet.next()) { + ++containsCnt; + } + assertEquals(1, containsCnt); + + int sinCnt = 0; + while (sinResultSet.next()) { + ++sinCnt; + } + assertEquals(1, sinCnt); + assertTestFail( + "select s1 from testUDTF where sin(s2)", + "The output type of the expression in WHERE clause should be BOOLEAN, actual data type: DOUBLE."); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCompareWithNull() { + tableResultSetEqualTest( + "select s1 from sg1 where s1 != null", new String[] {"s1"}, new String[] {}, DATABASE_NAME); + tableResultSetEqualTest( + "select s1 from sg1 where s1 <> null", new String[] {"s1"}, new String[] {}, DATABASE_NAME); + tableResultSetEqualTest( + "select s1 from sg1 where s1 = null", new String[] {"s1"}, new String[] {}, DATABASE_NAME); + } + + @Test + public void testCalculateWithNull() { + tableResultSetEqualTest( + "select s1 + null from sg1", + new String[] {"_col0"}, + new String[] {"null,", "null,"}, + DATABASE_NAME); + tableResultSetEqualTest( + "select s1 - null from sg1", + new String[] {"_col0"}, + new String[] {"null,", "null,"}, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBNestedQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBNestedQueryTableIT.java new file mode 100644 index 0000000000000..02de78ad0ad29 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBNestedQueryTableIT.java @@ -0,0 +1,480 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNestedQueryTableIT { + + private static final String DATABASE_NAME = "db"; + + protected static final int ITERATION_TIMES = 10; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setUdfMemoryBudgetInMB(5); + + EnvFactory.getEnv().initClusterEnvironment(); + createTable(); + generateData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void createTable() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "create table vehicle1(device_id STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 TEXT FIELD, s4 STRING FIELD, s5 DATE FIELD, s6 TIMESTAMP FIELD)"); + + statement.execute( + "create table vehicle2(device_id STRING TAG, s1 FLOAT FIELD, s2 DOUBLE FIELD, empty DOUBLE FIELD)"); + statement.execute( + "create table likeTest(device_id STRING TAG, s1 TEXT FIELD, s2 STRING FIELD)"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void generateData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + for (int i = 1; i <= ITERATION_TIMES; ++i) { + statement.execute( + String.format( + "insert into vehicle1(time,device_id,s1,s2,s3,s4,s6) values(%d,%s,%d,%d,%s,%s,%d)", + i, "'d1'", i, i, i, i, i)); + statement.execute( + (String.format( + "insert into vehicle2(time,device_id,s1,s2) values(%d,%s,%d,%d)", + i, "'d2'", i, i))); + } + statement.execute("insert into vehicle1(time,device_id,s5) values(1, 'd1', '2024-01-01')"); + statement.execute("insert into vehicle1(time,device_id,s5) values(2, 'd1','2024-01-02')"); + statement.execute("insert into vehicle1(time,device_id,s5) values(3, 'd1','2024-01-03')"); + statement.execute( + "insert into likeTest(time,device_id,s1,s2) values(1, 'd1','abcdef', '123456')"); + statement.execute( + "insert into likeTest(time,device_id,s1,s2) values(2, 'd1','_abcdef', '123\\456')"); + statement.execute( + "insert into likeTest(time,device_id,s1,s2) values(3, 'd1','abcdef%', '123#456')"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Ignore + @Test + public void testNestedRowByRowUDFExpressions() { + String sqlStr = + "select time, s1, s2, sin(sin(s1) * sin(s2) + cos(s1) * cos(s1)) + sin(sin(s1 - s1 + s2) * sin(s2) + cos(s1) * cos(s1)), asin(sin(asin(sin(s1 - s2 / (-s1))))) from vehicle2"; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + ResultSet resultSet = statement.executeQuery(sqlStr); + + assertEquals(1 + 4, resultSet.getMetaData().getColumnCount()); + + int count = 0; + while (resultSet.next()) { + ++count; + + assertEquals(count, resultSet.getLong(1)); + assertEquals(count, Double.parseDouble(resultSet.getString(2)), 0); + assertEquals(count, Double.parseDouble(resultSet.getString(3)), 0); + assertEquals(2 * Math.sin(1.0), Double.parseDouble(resultSet.getString(4)), 1e-5); + assertEquals( + Math.asin(Math.sin(Math.asin(Math.sin(count - count / (-count))))), + Double.parseDouble(resultSet.getString(5)), + 1e-5); + } + + assertEquals(ITERATION_TIMES, count); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testRawDataQueryWithConstants() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = "SELECT time, 1 + s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i + 1, rs.getInt(2), 0.01); + } + Assert.assertFalse(rs.next()); + } + + query = "SELECT time, (1 + 4) * 2 / 10 + s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i + 1, rs.getInt(2), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testDuplicatedRawDataQueryWithConstants() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = "SELECT time, 1 + s1, 1 + s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i + 1, rs.getInt(2), 0.01); + Assert.assertEquals(i + 1, rs.getInt(3), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testCommutativeLaws() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, s1, s1 + 1, 1 + s1, s1 * 2, 2 * s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i, rs.getInt(2)); + Assert.assertEquals(i + 1, rs.getInt(3), 0.01); + Assert.assertEquals(i + 1, rs.getInt(4), 0.01); + Assert.assertEquals(i * 2, rs.getInt(5), 0.01); + Assert.assertEquals(i * 2, rs.getInt(6), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testAssociativeLaws() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, s1, s1 + 1 + 2, (s1 + 1) + 2, s1 + (1 + 2), s1 * 2 * 3, s1 * (2 * 3), (s1 * 2) * 3 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i, rs.getInt(2)); + Assert.assertEquals(i + 3, rs.getInt(3), 0.01); + Assert.assertEquals(i + 3, rs.getInt(4), 0.01); + Assert.assertEquals(i + 3, rs.getInt(5), 0.01); + Assert.assertEquals(i * 6, rs.getInt(6), 0.01); + Assert.assertEquals(i * 6, rs.getInt(7), 0.01); + Assert.assertEquals(i * 6, rs.getInt(8), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testDistributiveLaw() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, s1, (s1 + 1) * 2, s1 * 2 + 1 * 2, (s1 + 1) / 2, s1 / 2 + 1 / 2 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i, rs.getInt(2)); + Assert.assertEquals(2 * i + 2, rs.getInt(3), 0.01); + Assert.assertEquals(2 * i + 2, rs.getInt(4), 0.01); + Assert.assertEquals((i + 1) / 2, rs.getInt(5), 0.01); + Assert.assertEquals(i / 2 + 1 / 2, rs.getInt(6), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testOrderOfArithmeticOperations() { + // Priority from high to low: + // 1. exponentiation and root extraction (not supported yet) + // 2. multiplication and division + // 3. addition and subtraction + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, 1 + s1 * 2 + 1, (1 + s1) * 2 + 1, (1 + s1) * (2 + 1) FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(2 * i + 2, rs.getInt(2), 0.01); + Assert.assertEquals(2 * i + 3, rs.getInt(3), 0.01); + Assert.assertEquals(3 * i + 3, rs.getInt(4), 0.01); + } + Assert.assertFalse(rs.next()); + } + + query = + "SELECT time, 1 - s1 / 2 + 1, (1 - s1) / 2 + 1, (1 - s1) / (2 + 1) FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(2 - i / 2, rs.getInt(2), 0.01); + Assert.assertEquals((1 - i) / 2 + 1, rs.getInt(3), 0.01); + Assert.assertEquals((1 - i) / (2 + 1), rs.getInt(4), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testBetweenExpression() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + int start = 2, end = 8; + statement.execute("USE " + DATABASE_NAME); + String query = + "SELECT * FROM vehicle1 where device_id='d1' and (s1 BETWEEN " + + start + + " AND " + + end + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM vehicle1 where device_id='d1' and (s1 NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(1, rs.getLong("time")); + Assert.assertEquals("1", rs.getString("s1")); + Assert.assertEquals("1", rs.getString("s2")); + Assert.assertEquals("1", rs.getString("s3")); + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM vehicle1 where device_id='d1' and (time BETWEEN " + + start + + " AND " + + end + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM vehicle1 where device_id='d1' and (time NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(1, rs.getLong("time")); + Assert.assertEquals("1", rs.getString("s1")); + Assert.assertEquals("1", rs.getString("s2")); + Assert.assertEquals("1", rs.getString("s3")); + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testRegularLikeInExpressions() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + String[] ans = new String[] {"abcdef"}; + String query = "SELECT s1 FROM likeTest where s1 LIKE 'abcdef'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"_abcdef"}; + query = "SELECT s1 FROM likeTest where s1 LIKE '\\_%' escape '\\'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"abcdef", "_abcdef", "abcdef%"}; + query = "SELECT s1 FROM likeTest where s1 LIKE '%abcde%' escape '\\'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 5; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"123456"}; + query = "SELECT s2 FROM likeTest where s2 LIKE '12345_'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"123\\456"}; + query = "SELECT s2 FROM likeTest where s2 LIKE '%\\\\%' escape '\\'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"123#456"}; + query = "SELECT s2 FROM likeTest where s2 LIKE '123##456' escape '#'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java new file mode 100644 index 0000000000000..e1571b001291d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java @@ -0,0 +1,705 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.common.collect.ImmutableSet; +import org.apache.tsfile.enums.TSDataType; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSimpleQueryTableIT { + private static final String DATABASE_NAME = "test"; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCreateTimeseries1() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD)"); + + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + if (resultSet.next() + && resultSet.getString(ColumnHeaderConstant.COLUMN_NAME).equals("s1")) { + assertEquals("INT32", resultSet.getString(ColumnHeaderConstant.DATATYPE).toUpperCase()); + assertEquals( + "FIELD", resultSet.getString(ColumnHeaderConstant.COLUMN_CATEGORY).toUpperCase()); + } + } + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Ignore // TODO After last query supported + @Test + public void testLastQueryNonCached() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute( + "create timeseries root.turbine.d1.s1 with datatype=FLOAT, encoding=GORILLA, compression=SNAPPY"); + statement.execute( + "create timeseries root.turbine.d1.s2 with datatype=FLOAT, encoding=GORILLA, compression=SNAPPY"); + statement.execute( + "create timeseries root.turbine.d2.s1 with datatype=FLOAT, encoding=GORILLA, compression=SNAPPY"); + statement.execute("insert into root.turbine.d1(timestamp,s1,s2) values(1,1,2)"); + + List expected = Arrays.asList("root.turbine.d1.s1", "root.turbine.d1.s2"); + List actual = new ArrayList<>(); + + try (ResultSet resultSet = statement.executeQuery("select last ** from root")) { + while (resultSet.next()) { + actual.add(resultSet.getString(ColumnHeaderConstant.TIMESERIES)); + } + } + + assertEquals(expected, actual); + + actual.clear(); + try (ResultSet resultSet = statement.executeQuery("select last * from root")) { + while (resultSet.next()) { + actual.add(resultSet.getString(ColumnHeaderConstant.TIMESERIES)); + } + } + + assertEquals(Collections.emptyList(), actual); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Ignore // TODO After Aggregation supported + @Test + public void testEmptyDataSet() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = statement.executeQuery("select * from root.**"); + // has an empty time column + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + try { + while (resultSet.next()) { + fail(); + } + + resultSet = + statement.executeQuery( + "select count(*) from root where time >= 1 and time <= 100 group by ([0, 100), 20ms, 20ms)"); + // has an empty time column + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + fail(); + } + + resultSet = statement.executeQuery("select count(*) from root"); + // has no column + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + fail(); + } + + resultSet = statement.executeQuery("select * from root.** align by device"); + // has time and device columns + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + fail(); + } + + resultSet = statement.executeQuery("select count(*) from root align by device"); + // has device column + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + fail(); + } + + resultSet = + statement.executeQuery( + "select count(*) from root where time >= 1 and time <= 100 " + + "group by ([0, 100), 20ms, 20ms) align by device"); + // has time and device columns + Assert.assertEquals(1, resultSet.getMetaData().getColumnCount()); + while (resultSet.next()) { + fail(); + } + } finally { + resultSet.close(); + } + + resultSet.close(); + } + } + + @Test + public void testOrderByTimeDesc() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s0 INT32 FIELD, s1 INT32 FIELD)"); + statement.execute("INSERT INTO table1(device, time, s0) VALUES ('d0', 1, 1)"); + statement.execute("INSERT INTO table1(device, time, s0) VALUES ('d0',2, 2)"); + statement.execute("INSERT INTO table1(device, time, s0) VALUES ('d0',3, 3)"); + statement.execute("INSERT INTO table1(device, time, s0) VALUES ('d0',4, 4)"); + statement.execute("INSERT INTO table1(device, time, s1) VALUES ('d0',3, 3)"); + statement.execute("INSERT INTO table1(device, time, s1) VALUES ('d0',1, 1)"); + statement.execute("flush"); + + String[] expectedHeader = new String[] {"time", "device", "s0", "s1"}; + String[] ret = + new String[] { + defaultFormatDataTime(4) + ",d0,4,null,", + defaultFormatDataTime(3) + ",d0,3,3,", + defaultFormatDataTime(2) + ",d0,2,null,", + defaultFormatDataTime(1) + ",d0,1,1,", + }; + + tableResultSetEqualTest( + "select * from table1 order by time desc", expectedHeader, ret, DATABASE_NAME); + } + } + + @Test + public void testShowTimeseriesDataSet1() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD)"); + + statement.execute("flush"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(12, count); + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testShowTimeseriesDataSet2() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(10); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD)"); + + statement.execute("flush"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(12, count); + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testShowTimeseriesDataSet3() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(15); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD)"); + + statement.execute("flush"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(12, count); + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testShowTimeseriesDataSet4() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD)"); + + statement.execute("flush"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1 limit 8")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(8, count); + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Ignore // TODO After describe limit offset supported + @Test + public void testDescribeWithLimitOffset() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD)"); + + Set exps = ImmutableSet.of("device", "s1"); + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1 offset 1 limit 2")) { + while (resultSet.next()) { + Assert.assertTrue(exps.contains(resultSet.getString(1))); + ++count; + } + } + Assert.assertEquals(2, count); + } + } + + @Ignore // TODO After DISTINCT supported + @Test + public void testShowDevicesWithLimitOffset() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO root.sg1.d0(timestamp, s1) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d1(timestamp, s2) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d2(timestamp, s3) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d3(timestamp, s4) VALUES (5, 5)"); + + List exps = Arrays.asList("root.sg1.d1,false", "root.sg1.d2,false"); + int count = 0; + try (ResultSet resultSet = statement.executeQuery("show devices limit 2 offset 1")) { + while (resultSet.next()) { + Assert.assertEquals( + exps.get(count), resultSet.getString(1) + "," + resultSet.getString(2)); + ++count; + } + } + Assert.assertEquals(2, count); + } + } + + @Ignore // TODO After DISTINCT supported + @Test + public void testShowDevicesWithLimit() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + List exps = Arrays.asList("root.sg1.d0,false", "root.sg1.d1,false"); + + statement.execute("INSERT INTO root.sg1.d0(timestamp, s1) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d1(timestamp, s2) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d2(timestamp, s3) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d3(timestamp, s4) VALUES (5, 5)"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("show devices limit 2")) { + while (resultSet.next()) { + Assert.assertEquals( + exps.get(count), resultSet.getString(1) + "," + resultSet.getString(2)); + ++count; + } + } + Assert.assertEquals(2, count); + } + } + + @Test + public void testFirstOverlappedPageFiltered() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s0 INT32 FIELD)"); + + // seq chunk : [1,10] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (1,'d0',1)"); + statement.execute("INSERT INTO table1(time, device, s0) VALUES (10,'d0',10)"); + + statement.execute("flush"); + + // seq chunk : [13,20] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (13,'d0',13)"); + statement.execute("INSERT INTO table1(time, device, s0) VALUES (20,'d0',20)"); + + statement.execute("flush"); + + // unseq chunk : [5,15] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (5,'d0',5)"); + statement.execute("INSERT INTO table1(time, device, s0) VALUES (15,'d0',15)"); + + statement.execute("flush"); + + long count = 0; + try (ResultSet resultSet = statement.executeQuery("select s0 from table1 where s0 > 18")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(1, count); + } + } + + @Test + public void testPartialInsertion() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s0 INT32 FIELD, s1 INT32 FIELD)"); + + try { + statement.execute("INSERT INTO table1(time, device, s0, s1) VALUES (1, 'd0', 1, 2.2)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("s1")); + } + + try (ResultSet resultSet = statement.executeQuery("select s0, s1 from table1")) { + while (resultSet.next()) { + assertEquals(1, resultSet.getInt("s0")); + assertEquals(null, resultSet.getString("s1")); + } + } + } + } + + @Test + public void testOverlappedPagesMerge() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s0 INT32 FIELD)"); + + // seq chunk : start-end [1000, 1000] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (1000, 'd0', 0)"); + + statement.execute("flush"); + + // unseq chunk : [1,10] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (1, 'd0', 1)"); + statement.execute("INSERT INTO table1(time, device, s0) VALUES (10, 'd0', 10)"); + + statement.execute("flush"); + + // usneq chunk : [5,15] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (5, 'd0', 5)"); + statement.execute("INSERT INTO table1(time, device, s0) VALUES (15, 'd0', 15)"); + + statement.execute("flush"); + + // unseq chunk : [15,15] + statement.execute("INSERT INTO table1(time, device, s0) VALUES (15, 'd0', 150)"); + + statement.execute("flush"); + + long count = 0; + + try (ResultSet resultSet = statement.executeQuery("select s0 from table1 where s0 < 100")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(4, count); + } + } + + @Ignore // TODO After delete data introduced + @Test + public void testUnseqUnsealedDeleteQuery() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s0 INT32 FIELD)"); + + // seq data + statement.execute("INSERT INTO table1(time, device, s0) VALUES (1000, 'd0', 1)"); + statement.execute("flush"); + + for (int i = 1; i <= 10; i++) { + statement.execute( + String.format("INSERT INTO table1(time, device, s0) VALUES (%d, 'd0', %d)", i, i)); + } + + statement.execute("flush"); + + // unseq data + for (int i = 11; i <= 20; i++) { + statement.execute( + String.format("INSERT INTO table1(time, device, s0) VALUES (%d, 'd0', %d)", i, i)); + } + + statement.execute("delete from table1 where time <= 15"); + + long count = 0; + + try (ResultSet resultSet = statement.executeQuery("select * from table1")) { + while (resultSet.next()) { + count++; + } + } + + System.out.println(count); + } + } + + @Test + public void testTimeseriesMetadataCache() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + StringBuilder createTableBuilder = new StringBuilder(); + createTableBuilder.append("CREATE TABLE table1(device STRING TAG,"); + for (int i = 0; i < 1000; i++) { + String columnName = "s" + i; + createTableBuilder.append(columnName).append(" INT32 FIELD,"); + } + createTableBuilder.deleteCharAt(createTableBuilder.lastIndexOf(",")).append(")"); + statement.execute(createTableBuilder.toString()); + for (int i = 1; i < 1000; i++) { + statement.execute("INSERT INTO table1(time, device, s" + i + ") VALUES (1000, 'd0', 1)"); + } + statement.execute("flush"); + statement.executeQuery("select s0 from table1"); + } catch (SQLException e) { + fail(); + } + } + + @Test + public void testUseSameStatement() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s0 INT32 FIELD, s1 INT32 FIELD)"); + statement.execute("insert into table1(time,device,s0,s1) values(1,'d0',1,1)"); + statement.execute("insert into table1(time,device,s0,s1) values(1000,'d1',1000,1000)"); + statement.execute("insert into table1(time,device,s0,s1) values(10,'d0',10,10)"); + + List resultSetList = new ArrayList<>(); + + ResultSet r1 = statement.executeQuery("select * from table1 where device='d0' and time <= 1"); + resultSetList.add(r1); + + ResultSet r2 = statement.executeQuery("select * from table1 where device='d1' and s0 = 1000"); + resultSetList.add(r2); + + ResultSet r3 = statement.executeQuery("select * from table1 where device='d0' and s1 = 10"); + resultSetList.add(r3); + + r1.next(); + Assert.assertEquals(r1.getLong(1), 1L); + Assert.assertEquals(r1.getLong(3), 1L); + Assert.assertEquals(r1.getLong(4), 1L); + + r2.next(); + Assert.assertEquals(r2.getLong(1), 1000L); + Assert.assertEquals(r2.getLong(3), 1000L); + Assert.assertEquals(r2.getLong(4), 1000L); + + r3.next(); + Assert.assertEquals(r3.getLong(1), 10L); + Assert.assertEquals(r3.getLong(3), 10L); + Assert.assertEquals(r3.getLong(4), 10L); + } + } + + @Test + public void testStorageGroupWithHyphenInName() { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE group_with_hyphen"); + } catch (SQLException e) { + fail(); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + while (resultSet.next()) { + final StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getString(1)); + Assert.assertTrue( + builder.toString().equals("group_with_hyphen") + || builder.toString().equals("information_schema")); + } + } + } catch (final SQLException e) { + fail(); + } + } + + @Test + public void testEnableAlign() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD)"); + ResultSet resultSet = statement.executeQuery("select time, s1, s2 from table1"); + ResultSetMetaData metaData = resultSet.getMetaData(); + int[] types = {Types.TIMESTAMP, Types.INTEGER, Types.BOOLEAN}; + int columnCount = metaData.getColumnCount(); + for (int i = 0; i < columnCount; i++) { + Assert.assertEquals(types[i], metaData.getColumnType(i + 1)); + } + } + } + + @Test + public void testNewDataType() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING FIELD)"); + + for (int i = 1; i <= 10; i++) { + statement.execute( + String.format( + "insert into table1(time, device, s4, s5, s6, s7) values(%d, 'd1', '%s', %d, %s, '%s')", + i, LocalDate.of(2024, 5, i % 31 + 1), i, "X'cafebabe'", i)); + } + + try (ResultSet resultSet = statement.executeQuery("select * from table1")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + final int columnCount = metaData.getColumnCount(); + assertEquals(6, columnCount); + HashMap columnType = new HashMap<>(); + for (int i = 3; i <= columnCount; i++) { + if (metaData.getColumnLabel(i).equals("s4")) { + columnType.put(i, TSDataType.DATE); + } else if (metaData.getColumnLabel(i).equals("s5")) { + columnType.put(i, TSDataType.TIMESTAMP); + } else if (metaData.getColumnLabel(i).equals("s6")) { + columnType.put(i, TSDataType.BLOB); + } else if (metaData.getColumnLabel(i).equals("s7")) { + columnType.put(i, TSDataType.TEXT); + } + } + byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (resultSet.next()) { + long time = resultSet.getLong(1); + Date date = resultSet.getDate(3); + long timestamp = resultSet.getLong(4); + byte[] blob = resultSet.getBytes(5); + String text = resultSet.getString(6); + assertEquals(2024 - 1900, date.getYear()); + assertEquals(5 - 1, date.getMonth()); + assertEquals(time % 31 + 1, date.getDate()); + assertEquals(time, timestamp); + assertArrayEquals(byteArray, blob); + assertEquals(String.valueOf(time), text); + } + } + + } catch (SQLException e) { + fail(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTable2IT.java new file mode 100644 index 0000000000000..c994d5807e121 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTable2IT.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceTable2IT extends IoTDBAlignByDeviceTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTable3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTable3IT.java new file mode 100644 index 0000000000000..afae14ad746e7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTable3IT.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceTable3IT extends IoTDBAlignByDeviceTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java new file mode 100644 index 0000000000000..3a882b7118c47 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java @@ -0,0 +1,551 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceTableIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create table vehicle(device_id STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 STRING FIELD, s4 BOOLEAN FIELD)", + "insert into vehicle(time, device_id, s0) values(1,'d0', 101)", + "insert into vehicle(time, device_id, s0) values(2,'d0', 198)", + "insert into vehicle(time, device_id, s0) values(100,'d0', 99)", + "insert into vehicle(time, device_id, s0) values(101,'d0', 99)", + "insert into vehicle(time, device_id, s0) values(102,'d0', 80)", + "insert into vehicle(time, device_id, s0) values(103,'d0', 99)", + "insert into vehicle(time, device_id, s0) values(104,'d0', 90)", + "insert into vehicle(time, device_id, s0) values(105,'d0', 99)", + "insert into vehicle(time, device_id, s0) values(106,'d0', 99)", + "insert into vehicle(time, device_id, s0) values(2,'d0', 10000)", + "insert into vehicle(time, device_id, s0) values(50,'d0', 10000)", + "insert into vehicle(time, device_id, s0) values(1000,'d0', 22222)", + "insert into vehicle(time, device_id, s1) values(1,'d0', 1101)", + "insert into vehicle(time, device_id, s1) values(2,'d0', 198)", + "insert into vehicle(time, device_id, s1) values(100,'d0', 199)", + "insert into vehicle(time, device_id, s1) values(101,'d0', 199)", + "insert into vehicle(time, device_id, s1) values(102,'d0', 180)", + "insert into vehicle(time, device_id, s1) values(103,'d0', 199)", + "insert into vehicle(time, device_id, s1) values(104,'d0', 190)", + "insert into vehicle(time, device_id, s1) values(105,'d0', 199)", + "insert into vehicle(time, device_id, s1) values(2,'d0', 40000)", + "insert into vehicle(time, device_id, s1) values(50,'d0', 50000)", + "insert into vehicle(time, device_id, s1) values(1000,'d0', 55555)", + "insert into vehicle(time, device_id, s1) values(2000-01-01T00:00:00+08:00,'d0', 100)", + "insert into vehicle(time, device_id, s2) values(1000,'d0', 55555)", + "insert into vehicle(time, device_id, s2) values(2,'d0', 2.22)", + "insert into vehicle(time, device_id, s2) values(3,'d0', 3.33)", + "insert into vehicle(time, device_id, s2) values(4,'d0', 4.44)", + "insert into vehicle(time, device_id, s2) values(102,'d0', 10.00)", + "insert into vehicle(time, device_id, s2) values(105,'d0', 11.11)", + "insert into vehicle(time, device_id, s2) values(1000,'d0', 1000.11)", + "insert into vehicle(time, device_id, s3) values(60,'d0', 'aaaaa')", + "insert into vehicle(time, device_id, s3) values(70,'d0', 'bbbbb')", + "insert into vehicle(time, device_id, s3) values(80,'d0', 'ccccc')", + "insert into vehicle(time, device_id, s3) values(101,'d0', 'ddddd')", + "insert into vehicle(time, device_id, s3) values(102,'d0', 'fffff')", + "insert into vehicle(time, device_id, s3) values(2000-01-01T00:00:00+08:00,'d0', 'good')", + "insert into vehicle(time, device_id, s4) values(100,'d0', false)", + "insert into vehicle(time, device_id, s4) values(100,'d0', true)", + "insert into vehicle(time, device_id, s0) values(1,'d1', 999)", + "insert into vehicle(time, device_id, s0) values(1000,'d1', 888)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,null,null,", + "1970-01-01T00:00:00.060Z,d0,null,null,null,aaaaa,null,", + "1970-01-01T00:00:00.070Z,d0,null,null,null,bbbbb,null,", + "1970-01-01T00:00:00.080Z,d0,null,null,null,ccccc,null,", + "1970-01-01T00:00:00.100Z,d0,99,199,null,null,true,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,ddddd,null,", + "1970-01-01T00:00:00.102Z,d0,80,180,10.0,fffff,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,null,null,", + "1970-01-01T00:00:00.104Z,d0,90,190,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + "1970-01-01T00:00:00.106Z,d0,99,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,null,null,", + "1999-12-31T16:00:00.000Z,d0,null,100,null,good,null,", + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle order by device_id", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void selectTestWithLimitOffset1() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle order by time asc, device_id offset 1 limit 5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectTestWithLimitOffset2() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1999-12-31T16:00:00.000Z,d0,null,100,null,good,null,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,null,null,", + "1970-01-01T00:00:00.106Z,d0,99,null,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle order by device_id desc, time desc offset 1 limit 5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWithDuplicatedPathsTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s0", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,101,1101,", + "1970-01-01T00:00:00.002Z,d0,10000,10000,40000,", + "1970-01-01T00:00:00.003Z,d0,null,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,10000,50000,", + "1970-01-01T00:00:00.060Z,d0,null,null,null,", + "1970-01-01T00:00:00.070Z,d0,null,null,null,", + "1970-01-01T00:00:00.080Z,d0,null,null,null,", + "1970-01-01T00:00:00.100Z,d0,99,99,199,", + "1970-01-01T00:00:00.101Z,d0,99,99,199,", + "1970-01-01T00:00:00.102Z,d0,80,80,180,", + "1970-01-01T00:00:00.103Z,d0,99,99,199,", + "1970-01-01T00:00:00.104Z,d0,90,90,190,", + "1970-01-01T00:00:00.105Z,d0,99,99,199,", + "1970-01-01T00:00:00.106Z,d0,99,99,null,", + "1970-01-01T00:00:01.000Z,d0,22222,22222,55555,", + "1999-12-31T16:00:00.000Z,d0,null,null,100,", + "1970-01-01T00:00:00.001Z,d1,999,999,null,", + "1970-01-01T00:00:01.000Z,d1,888,888,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectLimitTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s0", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d0,10000,10000,40000,", + "1970-01-01T00:00:00.003Z,d0,null,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,10000,50000,", + "1970-01-01T00:00:00.060Z,d0,null,null,null,", + "1970-01-01T00:00:00.070Z,d0,null,null,null,", + "1970-01-01T00:00:00.080Z,d0,null,null,null,", + "1970-01-01T00:00:00.100Z,d0,99,99,199,", + "1970-01-01T00:00:00.101Z,d0,99,99,199,", + "1970-01-01T00:00:00.102Z,d0,80,80,180,", + }; + tableResultSetEqualTest( + "select time, device_id, s0,s0,s1 from vehicle order by device_id,time offset 1 limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWithValueFilterTest() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.100Z,d0,99,199,null,null,true,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,ddddd,null,", + "1970-01-01T00:00:00.102Z,d0,80,180,10.0,fffff,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,null,null,", + "1970-01-01T00:00:00.104Z,d0,90,190,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle where s0 > 0 AND s1 < 200 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectDifferentSeriesWithValueFilterWithoutCacheTest() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.100Z,d0,99,", + "1970-01-01T00:00:00.101Z,d0,99,", + "1970-01-01T00:00:00.102Z,d0,80,", + "1970-01-01T00:00:00.103Z,d0,99,", + "1970-01-01T00:00:00.104Z,d0,90,", + "1970-01-01T00:00:00.105Z,d0,99,", + "1999-12-31T16:00:00.000Z,d0,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s0 from vehicle where s1 < 200 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectDifferentSeriesWithBinaryValueFilterWithoutCacheTest() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.105Z,d0,99,", + }; + tableResultSetEqualTest( + "select time, device_id, s0 from vehicle where s1 < 200 and s2 > 10 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void predicateCannotNormalizedTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,", + "1970-01-01T00:00:00.100Z,d0,99,199,null,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,", + }; + tableResultSetEqualTest( + "select time, device_id, s0,s1,s2 from vehicle where (((\"time\" > 10) AND (\"s1\" > 190)) OR (\"s2\" > 190.0) OR ((\"time\" < 4) AND (\"s1\" > 100))) order by device_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void duplicateProjectionsTest() { + String[] expectedHeader = new String[] {"Time", "device_id", "_col2", "_col3", "alias"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,1102,1102,1102,", + "1970-01-01T00:00:00.002Z,d0,40001,40001,40001,", + "1970-01-01T00:00:00.050Z,d0,50001,50001,50001,", + "1970-01-01T00:00:00.100Z,d0,200,200,200,", + "1970-01-01T00:00:00.101Z,d0,200,200,200,", + "1970-01-01T00:00:00.103Z,d0,200,200,200,", + "1970-01-01T00:00:00.105Z,d0,200,200,200,", + "1970-01-01T00:00:01.000Z,d0,55556,55556,55556,", + }; + tableResultSetEqualTest( + "select Time, device_id, s1+1, s1+1, s1+1 as alias from vehicle where (((\"time\" > 10) AND (\"s1\" > 190)) OR (\"s2\" > 190.0) OR ((\"time\" < 4) AND (\"s1\" > 100))) order by device_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // + // @Test + // public void aggregateTest() { + // String[] retArray = + // new String[] {"root.vehicle.d0,11,11,6,6,1,", "root.vehicle.d1,2,null,null,null,null,"}; + // + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // + // try (ResultSet resultSet = + // statement.executeQuery( + // "select count(s0),count(s1),count(s2),count(s3),count(s4) " + // + "from root.vehicle.d1,root.vehicle.d0 align by device")) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // List actualIndexToExpectedIndexList = + // checkHeader( + // resultSetMetaData, + // "Device,count(s0),count(s1),count(s2),count(s3),count(s4)", + // new int[] { + // Types.VARCHAR, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // }); + // + // int cnt = 0; + // while (resultSet.next()) { + // String[] expectedStrings = retArray[cnt].split(","); + // StringBuilder expectedBuilder = new StringBuilder(); + // StringBuilder actualBuilder = new StringBuilder(); + // for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + // actualBuilder.append(resultSet.getString(i)).append(","); + // expectedBuilder + // .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + // .append(","); + // } + // Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + // cnt++; + // } + // Assert.assertEquals(retArray.length, cnt); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupBytimeTest() { + // String[] retArray = + // new String[] { + // "2,root.vehicle.d0,1,1,3,0,0,", + // "22,root.vehicle.d0,0,0,0,0,0,", + // "42,root.vehicle.d0,0,0,0,0,0,", + // "2,root.vehicle.d1,0,null,null,null,null,", + // "22,root.vehicle.d1,0,null,null,null,null,", + // "42,root.vehicle.d1,0,null,null,null,null," + // }; + // + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // + // try (ResultSet resultSet = + // statement.executeQuery( + // "select count(*) from root.vehicle.** GROUP BY ([2,50),20ms) align by device")) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // List actualIndexToExpectedIndexList = + // checkHeader( + // resultSetMetaData, + // "time,Device,count(s0),count(s1),count(s2),count(s3),count(s4)", + // new int[] { + // Types.tIMESTAMP, + // Types.VARCHAR, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // }); + // + // int cnt = 0; + // while (resultSet.next()) { + // String[] expectedStrings = retArray[cnt].split(","); + // StringBuilder expectedBuilder = new StringBuilder(); + // StringBuilder actualBuilder = new StringBuilder(); + // for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + // actualBuilder.append(resultSet.getString(i)).append(","); + // expectedBuilder + // .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + // .append(","); + // } + // Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + // cnt++; + // } + // Assert.assertEquals(retArray.length, cnt); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupBytimeWithValueFilterTest() { + // String[] retArray = + // new String[] { + // "2,root.vehicle.d0,2,", "102,root.vehicle.d0,1", + // }; + // + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // + // try (ResultSet resultSet = + // statement.executeQuery( + // "select count(s2) from root.vehicle.d0 where s2 > 3 and s2 <= 10 GROUP BY + // ([2,200),100ms) align by device")) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // List actualIndexToExpectedIndexList = + // checkHeader( + // resultSetMetaData, + // "time,Device,count(s2)", + // new int[] { + // Types.tIMESTAMP, Types.VARCHAR, Types.BIGINT, + // }); + // + // int cnt = 0; + // while (resultSet.next()) { + // String[] expectedStrings = retArray[cnt].split(","); + // StringBuilder expectedBuilder = new StringBuilder(); + // StringBuilder actualBuilder = new StringBuilder(); + // for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + // actualBuilder.append(resultSet.getString(i)).append(","); + // expectedBuilder + // .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + // .append(","); + // } + // Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + // cnt++; + // } + // Assert.assertEquals(retArray.length, cnt); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + + @Test + public void unusualCaseTest2() { + + String[] expectedHeader = + new String[] {"s0", "s0", "s1", "time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "101,101,1101,1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + "10000,10000,40000,1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "null,null,null,1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "null,null,null,1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "999,999,null,1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + }; + tableResultSetEqualTest( + "select s0,s0,s1,* from vehicle where time < 20 and (device_id='d0' or device_id='d1') order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWithRegularExpressionTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,null,null,", + "1970-01-01T00:00:00.060Z,d0,null,null,null,aaaaa,null,", + "1970-01-01T00:00:00.070Z,d0,null,null,null,bbbbb,null,", + "1970-01-01T00:00:00.080Z,d0,null,null,null,ccccc,null,", + "1970-01-01T00:00:00.100Z,d0,99,199,null,null,true,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,ddddd,null,", + "1970-01-01T00:00:00.102Z,d0,80,180,10.0,fffff,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,null,null,", + "1970-01-01T00:00:00.104Z,d0,90,190,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + "1970-01-01T00:00:00.106Z,d0,99,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,null,null,", + "1999-12-31T16:00:00.000Z,d0,null,100,null,good,null,", + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle where device_id like 'd%' order by device_id", + expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void selectWithNonExistMeasurementInWhereClause() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle where s1=1101 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTable2IT.java new file mode 100644 index 0000000000000..a78b1e35780e0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTable2IT.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceWithTemplateTable2IT extends IoTDBAlignByDeviceWithTemplateTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java new file mode 100644 index 0000000000000..bf9e7f89db0cf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java @@ -0,0 +1,513 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceWithTemplateTableIT { + + private static final String DATABASE_NAME = "db"; + private static final String[] sqls = + new String[] { + "CREATE database " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create table table1(device_id STRING TAG, s1 FLOAT FIELD, s2 BOOLEAN FIELD, s3 INT32 FIELD)", + "INSERT INTO table1(Time, device_id, s1, s2, s3) VALUES (1,'d1', 1.1, false, 1), (2, 'd1', 2.2, false, 2), (1,'d2', 11.1, false, 11), (2,'d2', 22.2, false, 22), (1,'d3', 111.1, true, null), (4,'d3', 444.4, true, 44), (1,'d4', 1111.1, true, 1111), (5,'d4', 5555.5, false, 5555)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void selectWildcardNoFilterTest() { + // 1. order by device_id + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,1.1,false,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,false,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,false,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,false,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,true,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,true,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,true,", + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s1, s2 FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3", "s1"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,1.1,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,2.2,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,11.1,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,22.2,", + "1970-01-01T00:00:00.001Z,d3,111.1,true,null,111.1,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,444.4,", + "1970-01-01T00:00:00.001Z,d4,1111.1,true,1111,1111.1,", + "1970-01-01T00:00:00.005Z,d4,5555.5,false,5555,5555.5,", + }; + tableResultSetEqualTest( + "SELECT *, s1 FROM table1 order by device_id", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 where device_id = 'd1' or device_id = 'd2' or device_id = 'd6' order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by device_id + limit/offset + expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 order by device_id, time OFFSET 1 LIMIT 2", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by time + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555.5,false,5555,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + "1970-01-01T00:00:00.001Z,d3,111.1,true,null,", + "1970-01-01T00:00:00.001Z,d4,1111.1,true,1111,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY time DESC, device_id ASC", + expectedHeader, + retArray, + DATABASE_NAME); + + // 4. order by time + limit/offset + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555.5,false,5555,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY time DESC, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectMeasurementNoFilterTest() { + // 1. order by device_id + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,1.1,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,", + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s1 FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery("SELECT s3,s1,s_null FROM table1 order by device_id")) { + fail("should throw exception to indicate that s_null doesn't exist"); + } catch (SQLException e) { + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // 2. order by device + limit/offset + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,2.2,", "1970-01-01T00:00:00.001Z,d2,11,11.1,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s1 FROM table1 order by device_id,time OFFSET 1 LIMIT 2", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by time + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,", + "1970-01-01T00:00:00.001Z,d1,1,1.1,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s1 FROM table1 ORDER BY TIME DESC, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 4. order by time + limit/offset + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s1 FROM table1 ORDER BY time DESC, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWildcardWithFilterOrderByTimeTest() { + // 1. order by time + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE time < 5 ORDER BY TIME desc, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by time + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY time DESC, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by time + value filter: s_null > 1 + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery("SELECT * FROM table1 WHERE s_null > 1 order by device_id")) { + fail("should throw exception to indicate that s_null doesn't exist"); + } catch (SQLException e) { + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectWildcardWithFilterOrderByDeviceTest() { + // 1. order by device + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d4,1111.1,true,1111,", + "1970-01-01T00:00:00.001Z,d3,111.1,true,null,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE time < 5 ORDER BY device_id DESC, time LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by device + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY device_id DESC", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectMeasurementWithFilterOrderByTimeTest() { + // 1. order by time + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,44,true,", + "1970-01-01T00:00:00.002Z,d1,2,false,", + "1970-01-01T00:00:00.002Z,d2,22,false,", + "1970-01-01T00:00:00.001Z,d1,1,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s2 FROM table1 WHERE time < 5 ORDER BY time DESC, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by time + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,44,true,", "1970-01-01T00:00:00.002Z,d2,22,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s2 FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY time DESC, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectMeasurementWithFilterOrderByDeviceTest() { + // 1. order by device + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d4,1111,true,", + "1970-01-01T00:00:00.001Z,d3,null,true,", + "1970-01-01T00:00:00.004Z,d3,44,true,", + "1970-01-01T00:00:00.001Z,d2,11,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s2 FROM table1 WHERE time < 5 ORDER BY device_id DESC, time LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by device + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,44,true,", "1970-01-01T00:00:00.002Z,d2,22,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s2 FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY device_id DESC, time asc", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void aliasTest() { + String[] expectedHeader = new String[] {"aa", "bb", "s3", "s2", "time", "device_id"}; + + String[] retArray = + new String[] { + "1.1,false,1,false,1970-01-01T00:00:00.001Z,d1,", + "2.2,false,2,false,1970-01-01T00:00:00.002Z,d1,", + "11.1,false,11,false,1970-01-01T00:00:00.001Z,d2,", + "22.2,false,22,false,1970-01-01T00:00:00.002Z,d2,", + "111.1,true,null,true,1970-01-01T00:00:00.001Z,d3,", + "444.4,true,44,true,1970-01-01T00:00:00.004Z,d3,", + "1111.1,true,1111,true,1970-01-01T00:00:00.001Z,d4,", + "5555.5,false,5555,false,1970-01-01T00:00:00.005Z,d4,", + }; + tableResultSetEqualTest( + "SELECT s1 as aa, s2 as bb, s3, s2, time, device_id FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "a", "b"}; + + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1.1,1.1,", + "1970-01-01T00:00:00.002Z,d1,2.2,2.2,", + "1970-01-01T00:00:00.001Z,d2,11.1,11.1,", + "1970-01-01T00:00:00.002Z,d2,22.2,22.2,", + "1970-01-01T00:00:00.001Z,d3,111.1,111.1,", + "1970-01-01T00:00:00.004Z,d3,444.4,444.4,", + "1970-01-01T00:00:00.001Z,d4,1111.1,1111.1,", + "1970-01-01T00:00:00.005Z,d4,5555.5,5555.5,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s1 as a, s1 as b FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void orderByExpressionTest() { + // 1. order by basic measurement + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,false,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,false,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,false,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,false,", + "1970-01-01T00:00:00.001Z,d1,1,1.1,false,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,true,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,true,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,true,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s1, s2 FROM table1 order by s2 asc, s1 desc, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. select measurement is different with order by measurement + expectedHeader = new String[] {"time", "device_id", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,", + "1970-01-01T00:00:00.002Z,d2,22,", + "1970-01-01T00:00:00.001Z,d2,11,", + "1970-01-01T00:00:00.002Z,d1,2,", + "1970-01-01T00:00:00.001Z,d1,1,", + "1970-01-01T00:00:00.001Z,d4,1111,", + "1970-01-01T00:00:00.004Z,d3,44,", + "1970-01-01T00:00:00.001Z,d3,null,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3 FROM table1 order by s2 asc, s1 desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by expression + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,", + "1970-01-01T00:00:00.001Z,d4,1111,", + "1970-01-01T00:00:00.004Z,d3,44,", + "1970-01-01T00:00:00.002Z,d2,22,", + "1970-01-01T00:00:00.001Z,d2,11,", + "1970-01-01T00:00:00.002Z,d1,2,", + "1970-01-01T00:00:00.001Z,d1,1,", + "1970-01-01T00:00:00.001Z,d3,null,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3 FROM table1 order by s1+s3 desc", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void templateInvalidTest() { + + tableAssertTestFail( + "select s1 from table1 where s1", + "701: WHERE clause must evaluate to a boolean: actual type FLOAT", + DATABASE_NAME); + } + + @Test + public void emptyResultTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + + String[] retArray = new String[] {}; + tableResultSetEqualTest( + "SELECT * FROM table1 where time>=now()-1d and time<=now() " + "ORDER BY time DESC", + expectedHeader, + retArray, + DATABASE_NAME); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTable2IT.java new file mode 100644 index 0000000000000..a199836231045 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTable2IT.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByLimitOffsetAlignByDeviceTable2IT + extends IoTDBOrderByLimitOffsetAlignByDeviceTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData3(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableIT.java new file mode 100644 index 0000000000000..dbe72bea2cb6d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableIT.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByLimitOffsetAlignByDeviceTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData3(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void orderByCanNotPushLimitTest() { + // 1. value filter, can not push down LIMIT + String[] expectedHeader = new String[] {"time", "device_id", "s1"}; + String[] retArray = new String[] {"1970-01-01T00:00:00.003Z,d1,111,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE s1>40 ORDER BY Time, device_id LIMIT 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by expression, can not push down LIMIT + retArray = new String[] {"1970-01-01T00:00:00.003Z,d3,333,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY s1 DESC LIMIT 1", expectedHeader, retArray, DATABASE_NAME); + + // 3. time filter, can push down LIMIT + retArray = new String[] {"1970-01-01T00:00:00.002Z,d3,33,", "1970-01-01T00:00:00.002Z,d2,22,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE time>1 and time<3 ORDER BY device_id DESC,time LIMIT 2", + expectedHeader, + retArray, + DATABASE_NAME); + + // 4. both exist OFFSET and LIMIT, can push down LIMIT as OFFSET+LIMIT + retArray = new String[] {"1970-01-01T00:00:00.003Z,d2,222,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY Time DESC, device_id OFFSET 1 LIMIT 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // + // @Test + // public void aggregationWithHavingTest() { + // // when aggregation with having, can only use MergeSortNode but not use TopKNode + // String[] expectedHeader = new String[] {"Time,Device,sum(s1)"}; + // String[] retArray = new String[] {"3,root.db.d2,222.0,", "3,root.db.d3,333.0,"}; + // resultSetEqualTest( + // "select sum(s1) from root.db.** group by ((1,5],1ms) having(sum(s1)>111) order by time + // limit 2 align by device", + // expectedHeader, + // retArray); + // } + // + // @Test + // public void fillTest() { + // // linear fill can not use TopKNode + // String[] expectedHeader = new String[] {"Time,Device,s1,s2"}; + // String[] retArray = + // new String[] { + // "1,root.fill.d1,1,null,", + // "1,root.fill.d2,2,null,", + // "1,root.fill.d3,3,null,", + // "2,root.fill.d1,22,11.0,", + // }; + // resultSetEqualTest( + // "select * from root.fill.** order by time fill(linear) limit 4 align by device", + // expectedHeader, + // retArray); + // } + + private static final String DATABASE_NAME = "db"; + + private static final String DATABASE_FILL_NAME = "fill"; + + private static final String[] SQL_LIST = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table table1(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table1(Time, device_id, s1) VALUES(1, 'd1', 1), (2, 'd1', 11), (3, 'd1', 111)", + "INSERT INTO table1(Time, device_id, s1) VALUES(1, 'd2', 2), (2, 'd2', 22), (3, 'd2', 222)", + "INSERT INTO table1(Time, device_id, s1) VALUES(1, 'd3', 3), (2, 'd3', 33), (3, 'd3', 333)", + "CREATE DATABASE " + DATABASE_FILL_NAME, + "USE " + DATABASE_FILL_NAME, + "create table table1(device_id STRING TAG, s1 INT32 FIELD, s2 FLOAT FIELD)", + "INSERT INTO table1(Time,device_id,s1,s2) VALUES(1, 'd1', 1, null), (2, 'd1', null, 11), (3, 'd1', 111, 111.1)", + "INSERT INTO table1(Time,device_id,s1,s2) VALUES(1, 'd2', 2, null), (2, 'd2', 22, 22.2), (3, 'd2', 222, null)", + "INSERT INTO table1(Time,device_id,s1,s2) VALUES(1, 'd3', 3, null), (2, 'd3', 33, null), (3, 'd3', 333, 333.3)", + }; + + protected static void insertData3() { + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + for (String sql : SQL_LIST) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTable2IT.java new file mode 100644 index 0000000000000..0c75ced080993 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTable2IT.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByWithAlignByDeviceTable2IT extends IoTDBOrderByWithAlignByDeviceTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + insertData2(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTable3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTable3IT.java new file mode 100644 index 0000000000000..940712448aa77 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTable3IT.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByWithAlignByDeviceTable3IT extends IoTDBOrderByWithAlignByDeviceTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + insertData2(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java new file mode 100644 index 0000000000000..5c9ade9530b52 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java @@ -0,0 +1,1568 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.RpcUtils; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.db.it.utils.TestUtils.DEFAULT_ZONE_ID; +import static org.apache.iotdb.db.it.utils.TestUtils.TIME_PRECISION_IN_MS; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.rpc.RpcUtils.DEFAULT_TIME_FORMAT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByWithAlignByDeviceTableIT { + protected static final String DATABASE_NAME = "db"; + protected static final String[] places = + new String[] { + "London", + "Edinburgh", + "Belfast", + "Birmingham", + "Liverpool", + "Derby", + "Durham", + "Hereford", + "Manchester", + "Oxford" + }; + protected static final long startPrecipitation = 200; + protected static final double startTemperature = 20.0; + protected static final long startTime = 1668960000000L; + protected static final int numOfPointsInDevice = 20; + protected static final long timeGap = 100L; + protected static final Map deviceToStartTimestamp = new HashMap<>(); + public static final Map deviceToMaxTemperature = new HashMap<>(); + public static final Map deviceToAvgTemperature = new HashMap<>(); + public static final Map deviceToMaxPrecipitation = new HashMap<>(); + public static final Map deviceToAvgPrecipitation = new HashMap<>(); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + insertData2(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /** + * This method generate test data with crossing time. + * + *

The data can be viewed in online doc: + * + *

https://docs.google.com/spreadsheets/d/18XlOIi27ZIIdRnar2WNXVMxkZwjgwlPZmzJLVpZRpAA/edit#gid=0 + */ + protected static void insertData() { + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + statement.execute("create database " + DATABASE_NAME); + statement.execute("use " + DATABASE_NAME); + + statement.execute( + "create table weather(city STRING TAG, precipitation INT64 FIELD, temperature DOUBLE FIELD)"); + + // insert data + long start = startTime; + double[][] temperatures = new double[places.length][29]; + long[][] precipitations = new long[places.length][29]; + for (int index = 0; index < places.length; index++) { + String place = places[index]; + + for (int i = 0; i < numOfPointsInDevice; i++) { + long precipitation = startPrecipitation + place.hashCode() + (start + i * timeGap); + double temperature = startTemperature + place.hashCode() + (start + i * timeGap); + precipitations[index][(int) ((start - startTime) / timeGap) + i] = precipitation; + temperatures[index][(int) ((start - startTime) / timeGap) + i] = temperature; + String insertUniqueTime = + "INSERT INTO weather" + + "(Time,city,precipitation,temperature) VALUES(" + + (start + i * timeGap) + + "," + + String.format("'%s'", place) + + "," + + precipitation + + "," + + temperature + + ")"; + statement.addBatch(insertUniqueTime); + if (i == 0) { + deviceToStartTimestamp.put(place, start); + } + } + statement.executeBatch(); + statement.clearBatch(); + start += timeGap; + } + + for (int i = 0; i < places.length; i++) { + double[] aT = new double[3]; + double[] aP = new double[3]; + double[] mT = new double[3]; + long[] mP = new long[3]; + double totalTemperature = 0; + long totalPrecipitation = 0; + double maxTemperature = -1; + long maxPrecipitation = -1; + int cnt = 0; + for (int j = 0; j < precipitations[i].length; j++) { + totalTemperature += temperatures[i][j]; + totalPrecipitation += precipitations[i][j]; + maxPrecipitation = Math.max(maxPrecipitation, precipitations[i][j]); + maxTemperature = Math.max(maxTemperature, temperatures[i][j]); + if ((j + 1) % 10 == 0 || j == precipitations[i].length - 1) { + aT[cnt] = totalTemperature / 10; + aP[cnt] = (double) totalPrecipitation / 10; + mP[cnt] = maxPrecipitation; + mT[cnt] = maxTemperature; + cnt++; + totalTemperature = 0; + totalPrecipitation = 0; + maxTemperature = -1; + maxPrecipitation = -1; + } + } + deviceToMaxTemperature.put(places[i], mT); + deviceToMaxPrecipitation.put(places[i], mP); + deviceToAvgTemperature.put(places[i], aT); + deviceToAvgPrecipitation.put(places[i], aP); + } + + for (String sql : optimizedSQL) { + statement.execute(sql); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + // use to test if the compare result of time will overflow. + protected static void insertData2() { + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + statement.execute("create table overflow(device_id STRING TAG, value INT32 FIELD)"); + long startTime = 1; + for (int i = 0; i < 20; i++) { + String insertTime = + "INSERT INTO " + + "overflow" + + "(Time,device_id,value) VALUES(" + + (startTime + 2147483648L) + + "," + + "'virtual_device'" + + "," + + i + + ")"; + statement.addBatch(insertTime); + startTime += 2147483648L; + } + statement.executeBatch(); + statement.clearBatch(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void overFlowTest() { + + String[] expectedHeader = new String[] {"time", "value"}; + String[] retArray = new String[20]; + + long startTime = 1; + for (int i = 0; i < 20; i++) { + long time = startTime + 2147483648L; + retArray[i] = + String.format( + "%s,%d,", + RpcUtils.formatDatetime( + DEFAULT_TIME_FORMAT, TIME_PRECISION_IN_MS, time, DEFAULT_ZONE_ID), + i); + startTime += 2147483648L; + } + + tableResultSetEqualTest( + "SELECT time, value FROM overflow", expectedHeader, retArray, DATABASE_NAME); + } + + // ORDER BY DEVICE + @Test + public void orderByDeviceTest1() { + + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery("SELECT * FROM weather order by city")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(10, index); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void orderByDeviceTest2() { + + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + try (ResultSet resultSet = + statement.executeQuery("SELECT * FROM weather order by city asc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(10, index); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void orderByDeviceTest3() { + + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + String[] expectedDevice = + Arrays.stream(places.clone()).sorted(Comparator.reverseOrder()).toArray(String[]::new); + + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + try (ResultSet resultSet = + statement.executeQuery("SELECT * FROM weather order by city desc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(10, index); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ORDER BY TIME + + @Test + public void orderByTimeTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeTest1( + "SELECT * FROM weather ORDER BY time, city", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeTest1("SELECT * FROM weather ORDER BY time, city LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeTest2() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + orderByTimeTest1( + "SELECT * FROM weather ORDER BY TIME ASC, city", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeTest1( + "SELECT * FROM weather ORDER BY time ASC, city ASC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeTest3() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + orderByTimeTest3( + "SELECT * FROM weather ORDER BY time DESC, city ASC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeTest3( + "SELECT * FROM weather ORDER BY time DESC, city LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeExpressionTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeExpressionTest1( + "SELECT * FROM weather ORDER BY time DESC, precipitation DESC, city", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeExpressionTest1( + "SELECT * FROM weather ORDER BY time DESC, precipitation DESC, city asc LIMIT 100", + 100, + expectedHeader); + } + + @Test + public void orderExpressionTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByExpressionTest1( + "SELECT * FROM weather ORDER BY precipitation DESC, time DESC, city asc", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByExpressionTest1( + "SELECT * FROM weather ORDER BY precipitation DESC, time DESC, city limit 100", + 100, + expectedHeader); + } + + @Test + public void orderByDeviceTimeTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest1("SELECT * FROM weather ORDER BY city ASC,time DESC", 10, expectedHeader); + + orderByDeviceTimeTest1( + "SELECT * FROM weather ORDER BY city ASC,time DESC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByDeviceTimeTest2() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest2("SELECT * FROM weather ORDER BY city ASC,time ASC", 10, expectedHeader); + + orderByDeviceTimeTest2( + "SELECT * FROM weather ORDER BY city ASC,Time ASC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByDeviceTimeTest3() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest3( + "SELECT * FROM weather ORDER BY city DESC,time DESC", 10, expectedHeader); + + orderByDeviceTimeTest3( + "SELECT * FROM weather ORDER BY city DESC,time DESC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByDeviceTimeTest4() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest4("SELECT * FROM weather ORDER BY city DESC,time ASC", 10, expectedHeader); + + orderByDeviceTimeTest4( + "SELECT * FROM weather ORDER BY city DESC,time ASC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest1( + "SELECT * FROM weather ORDER BY time ASC,city DESC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest1( + "SELECT * FROM weather ORDER BY time ASC,city DESC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest2() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest2( + "SELECT * FROM weather ORDER BY time ASC,city ASC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest2( + "SELECT * FROM weather ORDER BY time ASC,city ASC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest3() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest3( + "SELECT * FROM weather ORDER BY time DESC,city DESC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest3( + "SELECT * FROM weather ORDER BY time DESC,city DESC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest4() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest4( + "SELECT * FROM weather ORDER BY time DESC,city ASC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest4( + "SELECT * FROM weather ORDER BY time DESC,city ASC LIMIT 100", 100, expectedHeader); + } + + public static void orderByTimeTest1(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = -1; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp >= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.isEmpty() && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeTest3(String sql, int count, String[] expectedHeader) { + + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeExpressionTest1(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = Long.MAX_VALUE; + long lastPrecipitation = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + if (actualDevice.equals(lastDevice) && actualTimeStamp == lastTimeStamp) { + assertTrue(actualPrecipitation <= lastPrecipitation); + lastPrecipitation = actualPrecipitation; + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByExpressionTest1(String sql, int count, String[] expectedHeader) { + + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = Long.MAX_VALUE; + long lastPrecipitation = Long.MAX_VALUE; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + assertTrue(actualPrecipitation <= lastPrecipitation); + if (actualPrecipitation == lastPrecipitation) { + assertTrue(actualTimeStamp <= lastTimeStamp); + } + lastPrecipitation = actualPrecipitation; + lastTimeStamp = actualTimeStamp; + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ORDER BY DEVICE,TIME + public static void orderByDeviceTimeTest1(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals( + deviceToStartTimestamp.get(actualDevice) + timeGap * (numOfPointsInDevice - cnt - 1), + actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByDeviceTimeTest2(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByDeviceTimeTest3(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = + Arrays.stream(places.clone()).sorted(Comparator.reverseOrder()).toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals( + deviceToStartTimestamp.get(actualDevice) + timeGap * (numOfPointsInDevice - cnt - 1), + actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByDeviceTimeTest4(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = + Arrays.stream(places.clone()).sorted(Comparator.reverseOrder()).toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ORDER BY TIME,DEVICE + public static void orderByTimeDeviceTest1(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = -1; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp >= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) <= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeDeviceTest2(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + long lastTimeStamp = -1; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp >= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeDeviceTest3(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + long lastTimeStamp = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) <= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeDeviceTest4(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + long lastTimeStamp = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // + // // aggregation query + // private static final int[][] countIn1000MSFiledWith100MSTimeGap = + // new int[][] { + // {10, 10, 0}, + // {9, 10, 1}, + // {8, 10, 2}, + // {7, 10, 3}, + // {6, 10, 4}, + // {5, 10, 5}, + // {4, 10, 6}, + // {3, 10, 7}, + // {2, 10, 8}, + // {1, 10, 9} + // }; + // + // private static int getCountNum(String device, int cnt) { + // int index = 0; + // for (int i = 0; i < places.length; i++) { + // if (places[i].equals(device)) { + // index = i; + // break; + // } + // } + // + // return countIn1000MSFiledWith100MSTimeGap[index][cnt]; + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest1() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000msC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime >= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt++; + // if (cnt % 3 == 0) { + // cnt = 0; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest2() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY DEVICE DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime >= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt++; + // if (cnt % 3 == 0) { + // cnt = 0; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest3() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY DEVICE + // DESC,TIME DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime <= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt--; + // if (cnt < 0) { + // cnt = 2; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest4() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY DEVICE + // ASC,TIME DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime <= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt--; + // if (cnt < 0) { + // cnt = 2; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest1() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime >= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt++; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest2() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = Long.MAX_VALUE; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime <= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt--; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest3() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME + // ASC,DEVICE DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime >= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt++; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest4() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME + // DESC,DEVICE DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = Long.MAX_VALUE; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime <= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt--; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + + // test the optimized plan + public static String[] optimizedSQL = + new String[] { + "create table optimize(plant_id STRING TAG, device_id STRING TAG, temperature DOUBLE FIELD, status BOOLEAN FIELD, hardware STRING FIELD)", + "insert into optimize(Time, plant_id, device_id, temperature, status) values(2017-11-01T00:00:00.000+08:00, 'wf01', 'wt01', 25.96, true)", + "insert into optimize(Time, plant_id, device_id, temperature, status) values(2017-11-01T00:01:00.000+08:00, 'wf01', 'wt01', 24.36, true)", + "insert into optimize(Time, plant_id, device_id, status, hardware) values(1970-01-01T08:00:00.001+08:00, 'wf02', 'wt02', true, 'v1')", + "insert into optimize(Time, plant_id, device_id, status, hardware) values(1970-01-01T08:00:00.002+08:00, 'wf02', 'wt02', false, 'v2')", + "insert into optimize(Time, plant_id, device_id, status, hardware) values(2017-11-01T00:00:00.000+08:00, 'wf02', 'wt02', false, 'v2')", + "insert into optimize(Time, plant_id, device_id, status, hardware) values(2017-11-01T00:01:00.000+08:00, 'wf02', 'wt02', true, 'v2')", + }; + + @Test + public void optimizedPlanTest() { + + String[] expectedHeader = + new String[] {"time", "plant_id", "device_id", "temperature", "status", "hardware"}; + String[] retArray = + new String[] { + "2017-10-31T16:01:00.000Z,wf02,wt02,null,true,v2,", + "2017-10-31T16:01:00.000Z,wf01,wt01,24.36,true,null,", + "2017-10-31T16:00:00.000Z,wf02,wt02,null,false,v2,", + "2017-10-31T16:00:00.000Z,wf01,wt01,25.96,true,null,", + "1970-01-01T00:00:00.002Z,wf02,wt02,null,false,v2,", + "1970-01-01T00:00:00.001Z,wf02,wt02,null,true,v1,", + }; + + tableResultSetEqualTest( + "SELECT * FROM optimize ORDER BY Time DESC,plant_id DESC,device_id desc", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedOffsetLimitPushDownTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedOffsetLimitPushDownTableIT.java new file mode 100644 index 0000000000000..739fb71398a57 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedOffsetLimitPushDownTableIT.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.relational.it.query.old.aligned.TableUtils.USE_DB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedOffsetLimitPushDownTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS db"); + statement.execute(USE_DB); + statement.execute( + "CREATE TABLE table0 (device string tag, s1 double field, s2 double field)"); + + statement.execute("insert into table0(device,time,s1,s2) values('d1',1,1,1)"); + statement.execute("insert into table0(device,time,s1,s2) values('d1',2,2,2)"); + statement.execute("insert into table0(device,time,s1,s2) values('d1',3,3,3)"); + statement.execute("insert into table0(device,time,s1,s2) values('d1',4,4,4)"); + statement.execute("insert into table0(device,time,s1,s2) values('d1',5,5,null)"); + statement.execute("insert into table0(device,time,s1,s2) values('d1',6,6,null)"); + statement.execute("insert into table0(device,time,s1,s2) values('d1',7,7,7)"); + + statement.execute("flush"); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void selectWithLimitPushDownTest1() { + + String[] retArray = new String[] {"3,3.0", "4,4.0"}; + + String[] columnNames = {"s1"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1 from table0 where device='d1' and time >= 3 limit 2")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectWithLimitPushDownTest2() { + + String[] retArray = new String[] {"6,6.0,null", "7,7.0,7.0"}; + + String[] columnNames = {"s1", "s2"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2 from table0 where device='d1' and time >= 1 offset 5")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable2IT.java new file mode 100644 index 0000000000000..dc70ac41941e7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable2IT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTable2IT extends IoTDBAlignedSeriesQueryTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(2); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable3IT.java new file mode 100644 index 0000000000000..dcaa4b9638f5a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable3IT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTable3IT extends IoTDBAlignedSeriesQueryTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable4IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable4IT.java new file mode 100644 index 0000000000000..40ac310e45f0d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable4IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTable4IT extends IoTDBAlignedSeriesQueryTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable5IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable5IT.java new file mode 100644 index 0000000000000..916934a0a0be4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTable5IT.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTable5IT extends IoTDBAlignedSeriesQueryTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(1) + .setMaxNumberOfPointsInPage(1) + .setDriverTaskExecutionTimeSliceInMs(20); + + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTableIT.java new file mode 100644 index 0000000000000..8ddb2a6223243 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBAlignedSeriesQueryTableIT.java @@ -0,0 +1,3698 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.db.utils.constant.TestConstant.avg; +import static org.apache.iotdb.db.utils.constant.TestConstant.count; +import static org.apache.iotdb.db.utils.constant.TestConstant.firstValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.lastValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.maxTime; +import static org.apache.iotdb.db.utils.constant.TestConstant.maxValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.minTime; +import static org.apache.iotdb.db.utils.constant.TestConstant.minValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.sum; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; +import static org.apache.iotdb.relational.it.query.old.aligned.TableUtils.USE_DB; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTableIT { + + private static final double DELTA = 1e-6; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ------------------------------Raw Query Without Value Filter---------------------------------- + @Test + public void selectAllAlignedWithoutValueFilterTest() { + + String[] retArray = + new String[] { + "1,1.0,1,1,true,aligned_test1", + "2,2.0,2,2,null,aligned_test2", + "3,30000.0,null,30000,true,aligned_unseq_test3", + "4,4.0,4,null,true,aligned_test4", + "5,5.0,5,null,true,aligned_test5", + "6,6.0,6,6,true,null", + "7,7.0,7,7,false,aligned_test7", + "8,8.0,8,8,null,aligned_test8", + "9,9.0,9,9,false,aligned_test9", + "10,null,10,10,true,aligned_test10", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + "21,null,null,21,true,null", + "22,null,null,22,true,null", + "23,230000.0,null,230000,false,null", + "24,null,null,24,true,null", + "25,null,null,25,true,null", + "26,null,null,26,false,null", + "27,null,null,27,false,null", + "28,null,null,28,false,null", + "29,null,null,29,false,null", + "30,null,null,30,false,null", + "31,null,31,null,null,aligned_test31", + "32,null,32,null,null,aligned_test32", + "33,null,33,null,null,aligned_test33", + "34,null,34,null,null,aligned_test34", + "35,null,35,null,null,aligned_test35", + "36,null,36,null,null,aligned_test36", + "37,null,37,null,null,aligned_test37", + "38,null,38,null,null,aligned_test38", + "39,null,39,null,null,aligned_test39", + "40,null,40,null,null,aligned_test40", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeFilterTest() { + + String[] retArray = + new String[] { + "9,9.0,9,9,false,aligned_test9", + "10,null,10,10,true,aligned_test10", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + "21,null,null,21,true,null", + "22,null,null,22,true,null", + "23,230000.0,null,230000,false,null", + "24,null,null,24,true,null", + "25,null,null,25,true,null", + "26,null,null,26,false,null", + "27,null,null,27,false,null", + "28,null,null,28,false,null", + "29,null,null,29,false,null", + "30,null,null,30,false,null", + "31,null,31,null,null,aligned_test31", + "32,null,32,null,null,aligned_test32", + "33,null,33,null,null,aligned_test33", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterTest1() { + + String[] retArray = + new String[] { + "1,1.0,true,aligned_test1", + "2,2.0,null,aligned_test2", + "3,30000.0,true,aligned_unseq_test3", + "4,4.0,true,aligned_test4", + "5,5.0,true,aligned_test5", + "6,6.0,true,null", + "7,7.0,false,aligned_test7", + "8,8.0,null,aligned_test8", + "9,9.0,false,aligned_test9", + "10,null,true,aligned_test10", + "11,11.0,null,null", + "12,12.0,null,null", + "13,130000.0,true,aligned_unseq_test13", + "14,14.0,null,null", + "15,15.0,null,null", + "16,16.0,null,null", + "17,17.0,null,null", + "18,18.0,null,null", + "19,19.0,null,null", + "20,20.0,null,null", + "21,null,true,null", + "22,null,true,null", + "23,230000.0,false,null", + "24,null,true,null", + "25,null,true,null", + "26,null,false,null", + "27,null,false,null", + "28,null,false,null", + "29,null,false,null", + "30,null,false,null", + "31,null,null,aligned_test31", + "32,null,null,aligned_test32", + "33,null,null,aligned_test33", + "34,null,null,aligned_test34", + "35,null,null,aligned_test35", + "36,null,null,aligned_test36", + "37,null,null,aligned_test37", + "38,null,null,aligned_test38", + "39,null,null,aligned_test39", + "40,null,null,aligned_test40", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery("select Time,s1,s4,s5 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterTest2() { + + String[] retArray = + new String[] { + "1,1.0,true", + "2,2.0,null", + "3,30000.0,true", + "4,4.0,true", + "5,5.0,true", + "6,6.0,true", + "7,7.0,false", + "8,8.0,null", + "9,9.0,false", + "10,null,true", + "11,11.0,null", + "12,12.0,null", + "13,130000.0,true", + "14,14.0,null", + "15,15.0,null", + "16,16.0,null", + "17,17.0,null", + "18,18.0,null", + "19,19.0,null", + "20,20.0,null", + "21,null,true", + "22,null,true", + "23,230000.0,false", + "24,null,true", + "25,null,true", + "26,null,false", + "27,null,false", + "28,null,false", + "29,null,false", + "30,null,false", + "31,null,null", + "32,null,null", + "33,null,null", + "34,null,null", + "35,null,null", + "36,null,null", + "37,null,null", + "38,null,null", + "39,null,null", + "40,null,null", + }; + + String[] columnNames = {"s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery("select Time,s1,s4 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithTimeFilterTest() { + + String[] retArray = + new String[] { + "16,16.0,null,null", + "17,17.0,null,null", + "18,18.0,null,null", + "19,19.0,null,null", + "20,20.0,null,null", + "21,null,true,null", + "22,null,true,null", + "23,230000.0,false,null", + "24,null,true,null", + "25,null,true,null", + "26,null,false,null", + "27,null,false,null", + "28,null,false,null", + "29,null,false,null", + "30,null,false,null", + "31,null,null,aligned_test31", + "32,null,null,aligned_test32", + "33,null,null,aligned_test33", + "34,null,null,aligned_test34", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4,s5 from table0 where device='d1' and time >= 16 and time <= 34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ------------------------------Raw Query With Value Filter------------------------------------- + @Test + public void selectAllAlignedWithValueFilterTest1() { + + String[] retArray = + new String[] { + "1,1.0,1,1,true,aligned_test1", + "3,30000.0,null,30000,true,aligned_unseq_test3", + "4,4.0,4,null,true,aligned_test4", + "5,5.0,5,null,true,aligned_test5", + "6,6.0,6,6,true,null", + "10,null,10,10,true,aligned_test10", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "21,null,null,21,true,null", + "22,null,null,22,true,null", + "24,null,null,24,true,null", + "25,null,null,25,true,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterTest2() { + + String[] retArray = + new String[] { + "12,12.0,12,12,null,null", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and s1 > 11.0 and s2 <= 33 order by time asc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterTest3() { + + String[] retArray = + new String[] { + "1,1.0,1,1,true,aligned_test1", + "2,2.0,2,2,null,aligned_test2", + "3,30000.0,null,30000,true,aligned_unseq_test3", + "4,4.0,4,null,true,aligned_test4", + "5,5.0,5,null,true,aligned_test5", + "6,6.0,6,6,true,null", + "7,7.0,7,7,false,aligned_test7", + "8,8.0,8,8,null,aligned_test8", + "9,9.0,9,9,false,aligned_test9", + "10,null,10,10,true,aligned_test10", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + "23,230000.0,null,230000,false,null", + "31,null,31,null,null,aligned_test31", + "32,null,32,null,null,aligned_test32", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and (s1 >= 13.0 or s2 < 33) order by time asc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeAndValueFilterTest1() { + + String[] retArray = + new String[] { + "9,9.0,9,9,false,aligned_test9", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 and s1 < 19.0")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithLimitOffsetTest() { + + String[] retArray = + new String[] { + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 offset 5 limit 5")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterTest1() { + + String[] retArray = + new String[] { + "1,1.0,true,aligned_test1", + "2,2.0,null,aligned_test2", + "4,4.0,true,aligned_test4", + "5,5.0,true,aligned_test5", + "6,6.0,true,null", + "7,7.0,false,aligned_test7", + "8,8.0,null,aligned_test8", + "9,9.0,false,aligned_test9", + "11,11.0,null,null", + "12,12.0,null,null", + "14,14.0,null,null", + "15,15.0,null,null", + "16,16.0,null,null", + "34,null,null,aligned_test34", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4,s5 from table0 where device='d1' and s1 < 17.0 or s5 = 'aligned_test34'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterTest2() { + + String[] retArray = + new String[] { + "7,7.0,false", "9,9.0,false", + }; + + String[] columnNames = {"s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4 from table0 where device='d1' and s1 < 19.0 and s4 = false")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithTimeAndValueFilterTest() { + + String[] retArray = + new String[] { + "23,230000.0,false,null", + "26,null,false,null", + "27,null,false,null", + "28,null,false,null", + "29,null,false,null", + "30,null,false,null", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4,s5 from table0 where device='d1' and time >= 16 and time <= 34 and s4=false")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // --------------------------Aggregation Query Without Value Filter--------------------------- + @Ignore + @Test + public void countAllAlignedWithoutTimeFilterTest() { + String[] retArray = new String[] {"20", "29", "28", "19", "20"}; + String[] columnNames = { + "count(d1.s1)", "count(d1.s2)", "count(d1.s3)", "count(d1.s4)", "count(d1.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = statement.executeQuery("select count(*) from d1")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedAndNonAlignedWithoutTimeFilterTest() { + String[] retArray = new String[] {"20", "29", "28", "19", "20", "19", "29", "28", "18", "19"}; + String[] columnNames = { + "count(d1.s1)", + "count(d1.s2)", + "count(d1.s3)", + "count(d1.s4)", + "count(d1.s5)", + "count(d2.s1)", + "count(d2.s2)", + "count(d2.s3)", + "count(d2.s4)", + "count(d2.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = statement.executeQuery("select count(*) from root.sg1.*")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithTimeFilterTest() { + String[] retArray = new String[] {"12", "15", "22", "13", "6"}; + String[] columnNames = { + "count(d1.s1)", "count(d1.s2)", "count(d1.s3)", "count(d1.s4)", "count(d1.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 where time >= 9 and time <= 33")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** aggregate multi columns of aligned timeseries in one SQL */ + @Ignore + @Test + public void aggregateSomeAlignedWithoutTimeFilterTest() { + double[] retArray = + new double[] { + 20, 29, 28, 390184, 130549, 390417, 19509.2, 4501.689655172413, 13943.464285714286 + }; + String[] columnNames = { + "count(d1.s1)", + "count(d1.s2)", + "count(d1.s3)", + "sum(d1.s1)", + "sum(d1.s2)", + "sum(d1.s3)", + "avg(d1.s1)", + "avg(d1.s2)", + "avg(d1.s3)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1),count(s2),count(s3),sum(s1),sum(s2),sum(s3),avg(s1),avg(s2),avg(s3) from d1")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = Double.parseDouble(resultSet.getString(index)); + } + assertArrayEquals(retArray, ans, DELTA); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** aggregate multi columns of aligned timeseries in one SQL */ + @Ignore + @Test + public void aggregateSomeAlignedWithTimeFilterTest() { + double[] retArray = + new double[] { + 6, 9, 15, 230090, 220, 230322, 38348.333333333336, 24.444444444444443, 15354.8 + }; + String[] columnNames = { + "count(d1.s1)", + "count(d1.s2)", + "count(d1.s3)", + "sum(d1.s1)", + "sum(d1.s2)", + "sum(d1.s3)", + "avg(d1.s1)", + "avg(d1.s2)", + "avg(d1.s3)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1),count (s2),count (s3),sum(s1),sum(s2),sum(s3),avg(s1),avg(s2),avg(s3) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = Double.parseDouble(resultSet.getString(index)); + } + assertArrayEquals(retArray, ans, DELTA); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countSingleAlignedWithTimeFilterTest() { + String[] retArray = new String[] {"9"}; + String[] columnNames = {"count(d1.s2)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(s2) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + // No need to add time column for aggregation query + for (String columnName : columnNames) { + int index = map.get(columnName); + if (builder.length() != 0) { + builder.append(","); + } + builder.append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void sumSingleAlignedWithTimeFilterTest() { + String[] retArray = new String[] {"230322.0"}; + String[] columnNames = {"sum(d1.s3)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select sum(s3) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + // No need to add time column for aggregation query + for (String columnName : columnNames) { + int index = map.get(columnName); + if (builder.length() != 0) { + builder.append(","); + } + builder.append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void avgSingleAlignedWithTimeFilterTest() { + double[][] retArray = {{24.444444444444443}}; + String[] columnNames = {"avg(d1.s2)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select avg(s2) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + StringBuilder builder = new StringBuilder(); + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = Double.parseDouble(resultSet.getString(index)); + } + assertArrayEquals(retArray[cnt], ans, DELTA); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // --------------------------------Aggregation with value filter------------------------------ + @Ignore + @Test + public void countAlignedWithValueFilterTest() { + String[] retArray = new String[] {"11"}; + String[] columnNames = {"count(d1.s4)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(s4) from d1 where s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationFuncAlignedWithValueFilterTest() throws ClassNotFoundException { + String[] retArray = new String[] {"8", "42.0", "5.25", "1.0", "9.0", "1", "9", "1.0", "9.0"}; + String[] columnNames = { + "count(d1.s1)", + "sum(d1.s1)", + "avg(d1.s1)", + "first_value(d1.s1)", + "last_value(d1.s1)", + "min_time(d1.s1)", + "max_time(d1.s1)", + "min_value(d1.s1)", + "max_value(d1.s1)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s1), avg(s1), " + + "first_value(s1), last_value(s1), " + + "min_time(s1), max_time(s1)," + + "max_value(s1), min_value(s1) from d1 where s1 < 10")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithValueFilterTest() throws ClassNotFoundException { + String[] retArray = new String[] {"6", "6", "9", "11", "6"}; + String[] columnNames = { + "count(d1.s1)", "count(d1.s2)", "count(d1.s3)", "count(d1.s4)", "count(d1.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 where s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationAllAlignedWithValueFilterTest() throws ClassNotFoundException { + String[] retArray = new String[] {"160016.0", "11", "1", "13"}; + String[] columnNames = { + "sum(d1.s1)", "count(d1.s4)", "min_value(d1.s3)", "max_time(d1.s2)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select sum(s1), count(s4), min_value(s3), max_time(s2) from d1 where s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ---------------------------------Align by device query ------------------------------------ + @Test + public void selectAllAlignedWithoutValueFilterAlignByDeviceTest() { + String[] retArray = + new String[] { + "1,d1,1.0,1,1,true,aligned_test1", + "2,d1,2.0,2,2,null,aligned_test2", + "3,d1,30000.0,null,30000,true,aligned_unseq_test3", + "4,d1,4.0,4,null,true,aligned_test4", + "5,d1,5.0,5,null,true,aligned_test5", + "6,d1,6.0,6,6,true,null", + "7,d1,7.0,7,7,false,aligned_test7", + "8,d1,8.0,8,8,null,aligned_test8", + "9,d1,9.0,9,9,false,aligned_test9", + "10,d1,null,10,10,true,aligned_test10", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "23,d1,230000.0,null,230000,false,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + "26,d1,null,null,26,false,null", + "27,d1,null,null,27,false,null", + "28,d1,null,null,28,false,null", + "29,d1,null,null,29,false,null", + "30,d1,null,null,30,false,null", + "31,d1,null,31,null,null,aligned_test31", + "32,d1,null,32,null,null,aligned_test32", + "33,d1,null,33,null,null,aligned_test33", + "34,d1,null,34,null,null,aligned_test34", + "35,d1,null,35,null,null,aligned_test35", + "36,d1,null,36,null,null,aligned_test36", + "37,d1,null,37,null,null,aligned_test37", + "38,d1,null,38,null,null,aligned_test38", + "39,d1,null,39,null,null,aligned_test39", + "40,d1,null,40,null,null,aligned_test40", + }; + + String[] columnNames = {"device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,device,s1,s2,s3,s4,s5 from table0 where device='d1' order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedAndNonAlignedAlignByDeviceTest() { + String[] retArray = + new String[] { + "1,d1,1.0,1,1,true,aligned_test1", + "2,d1,2.0,2,2,null,aligned_test2", + "3,d1,30000.0,null,30000,true,aligned_unseq_test3", + "4,d1,4.0,4,null,true,aligned_test4", + "5,d1,5.0,5,null,true,aligned_test5", + "6,d1,6.0,6,6,true,null", + "7,d1,7.0,7,7,false,aligned_test7", + "8,d1,8.0,8,8,null,aligned_test8", + "9,d1,9.0,9,9,false,aligned_test9", + "10,d1,null,10,10,true,aligned_test10", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "23,d1,230000.0,null,230000,false,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + "26,d1,null,null,26,false,null", + "27,d1,null,null,27,false,null", + "28,d1,null,null,28,false,null", + "29,d1,null,null,29,false,null", + "30,d1,null,null,30,false,null", + "31,d1,null,31,null,null,aligned_test31", + "32,d1,null,32,null,null,aligned_test32", + "33,d1,null,33,null,null,aligned_test33", + "34,d1,null,34,null,null,aligned_test34", + "35,d1,null,35,null,null,aligned_test35", + "36,d1,null,36,null,null,aligned_test36", + "37,d1,null,37,null,null,aligned_test37", + "38,d1,null,38,null,null,aligned_test38", + "39,d1,null,39,null,null,aligned_test39", + "40,d1,null,40,null,null,aligned_test40", + "1,d2,1.0,1,1,true,non_aligned_test1", + "2,d2,2.0,2,2,null,non_aligned_test2", + "3,d2,3.0,null,3,false,non_aligned_test3", + "4,d2,4.0,4,null,true,non_aligned_test4", + "5,d2,5.0,5,null,true,non_aligned_test5", + "6,d2,6.0,6,6,true,null", + "7,d2,7.0,7,7,false,non_aligned_test7", + "8,d2,8.0,8,8,null,non_aligned_test8", + "9,d2,9.0,9,9,false,non_aligned_test9", + "10,d2,null,10,10,true,non_aligned_test10", + "11,d2,11.0,11,11,null,null", + "12,d2,12.0,12,12,null,null", + "13,d2,13.0,13,13,null,null", + "14,d2,14.0,14,14,null,null", + "15,d2,15.0,15,15,null,null", + "16,d2,16.0,16,16,null,null", + "17,d2,17.0,17,17,null,null", + "18,d2,18.0,18,18,null,null", + "19,d2,19.0,19,19,null,null", + "20,d2,20.0,20,20,null,null", + "21,d2,null,null,21,true,null", + "22,d2,null,null,22,true,null", + "23,d2,null,null,23,true,null", + "24,d2,null,null,24,true,null", + "25,d2,null,null,25,true,null", + "26,d2,null,null,26,false,null", + "27,d2,null,null,27,false,null", + "28,d2,null,null,28,false,null", + "29,d2,null,null,29,false,null", + "30,d2,null,null,30,false,null", + "31,d2,null,31,null,null,non_aligned_test31", + "32,d2,null,32,null,null,non_aligned_test32", + "33,d2,null,33,null,null,non_aligned_test33", + "34,d2,null,34,null,null,non_aligned_test34", + "35,d2,null,35,null,null,non_aligned_test35", + "36,d2,null,36,null,null,non_aligned_test36", + "37,d2,null,37,null,null,non_aligned_test37", + "38,d2,null,38,null,null,non_aligned_test38", + "39,d2,null,39,null,null,non_aligned_test39", + "40,d2,null,40,null,null,non_aligned_test40", + }; + + String[] columnNames = {"device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,device,s1,s2,s3,s4,s5 from table0 order by device, time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeFilterAlignByDeviceTest() { + String[] retArray = + new String[] { + "9,d1,9.0,9,9,false,aligned_test9", + "10,d1,null,10,10,true,aligned_test10", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "23,d1,230000.0,null,230000,false,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + "26,d1,null,null,26,false,null", + "27,d1,null,null,27,false,null", + "28,d1,null,null,28,false,null", + "29,d1,null,null,29,false,null", + "30,d1,null,null,30,false,null", + "31,d1,null,31,null,null,aligned_test31", + "32,d1,null,32,null,null,aligned_test32", + "33,d1,null,33,null,null,aligned_test33", + }; + + String[] columnNames = {"device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,device,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "1,d1,1.0,true,aligned_test1", + "2,d1,2.0,null,aligned_test2", + "3,d1,30000.0,true,aligned_unseq_test3", + "4,d1,4.0,true,aligned_test4", + "5,d1,5.0,true,aligned_test5", + "6,d1,6.0,true,null", + "7,d1,7.0,false,aligned_test7", + "8,d1,8.0,null,aligned_test8", + "9,d1,9.0,false,aligned_test9", + "10,d1,null,true,aligned_test10", + "11,d1,11.0,null,null", + "12,d1,12.0,null,null", + "13,d1,130000.0,true,aligned_unseq_test13", + "14,d1,14.0,null,null", + "15,d1,15.0,null,null", + "16,d1,16.0,null,null", + "17,d1,17.0,null,null", + "18,d1,18.0,null,null", + "19,d1,19.0,null,null", + "20,d1,20.0,null,null", + "21,d1,null,true,null", + "22,d1,null,true,null", + "23,d1,230000.0,false,null", + "24,d1,null,true,null", + "25,d1,null,true,null", + "26,d1,null,false,null", + "27,d1,null,false,null", + "28,d1,null,false,null", + "29,d1,null,false,null", + "30,d1,null,false,null", + "31,d1,null,null,aligned_test31", + "32,d1,null,null,aligned_test32", + "33,d1,null,null,aligned_test33", + "34,d1,null,null,aligned_test34", + "35,d1,null,null,aligned_test35", + "36,d1,null,null,aligned_test36", + "37,d1,null,null,aligned_test37", + "38,d1,null,null,aligned_test38", + "39,d1,null,null,aligned_test39", + "40,d1,null,null,aligned_test40", + }; + + String[] columnNames = {"device", "s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery("select Time,device,s1,s4,s5 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterAlignByDeviceTest2() { + String[] retArray = + new String[] { + "1,d1,1.0,true", + "2,d1,2.0,null", + "3,d1,30000.0,true", + "4,d1,4.0,true", + "5,d1,5.0,true", + "6,d1,6.0,true", + "7,d1,7.0,false", + "8,d1,8.0,null", + "9,d1,9.0,false", + "10,d1,null,true", + "11,d1,11.0,null", + "12,d1,12.0,null", + "13,d1,130000.0,true", + "14,d1,14.0,null", + "15,d1,15.0,null", + "16,d1,16.0,null", + "17,d1,17.0,null", + "18,d1,18.0,null", + "19,d1,19.0,null", + "20,d1,20.0,null", + "21,d1,null,true", + "22,d1,null,true", + "23,d1,230000.0,false", + "24,d1,null,true", + "25,d1,null,true", + "26,d1,null,false", + "27,d1,null,false", + "28,d1,null,false", + "29,d1,null,false", + "30,d1,null,false", + "31,d1,null,null", + "32,d1,null,null", + "33,d1,null,null", + "34,d1,null,null", + "35,d1,null,null", + "36,d1,null,null", + "37,d1,null,null", + "38,d1,null,null", + "39,d1,null,null", + "40,d1,null,null", + }; + + String[] columnNames = {"Device", "s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery("select Time,Device,s1,s4 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithTimeFilterAlignByDeviceTest() { + String[] retArray = + new String[] { + "16,d1,16.0,null,null", + "17,d1,17.0,null,null", + "18,d1,18.0,null,null", + "19,d1,19.0,null,null", + "20,d1,20.0,null,null", + "21,d1,null,true,null", + "22,d1,null,true,null", + "23,d1,230000.0,false,null", + "24,d1,null,true,null", + "25,d1,null,true,null", + "26,d1,null,false,null", + "27,d1,null,false,null", + "28,d1,null,false,null", + "29,d1,null,false,null", + "30,d1,null,false,null", + "31,d1,null,null,aligned_test31", + "32,d1,null,null,aligned_test32", + "33,d1,null,null,aligned_test33", + "34,d1,null,null,aligned_test34", + }; + + String[] columnNames = {"Device", "s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s4,s5 from table0 where device='d1' and time >= 16 and time <= 34 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "1,d1,1.0,1,1,true,aligned_test1", + "3,d1,30000.0,null,30000,true,aligned_unseq_test3", + "4,d1,4.0,4,null,true,aligned_test4", + "5,d1,5.0,5,null,true,aligned_test5", + "6,d1,6.0,6,6,true,null", + "10,d1,null,10,10,true,aligned_test10", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + }; + + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s2,s3,s4,s5 from table0 where device='d1' and s4 = true order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterAlignByDeviceTest2() { + String[] retArray = + new String[] { + "12,d1,12.0,12,12,null,null", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + }; + + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s2,s3,s4,s5 from table0 where device='d1' and s1 > 11.0 and s2 <= 33 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeAndValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "9,d1,9.0,9,9,false,aligned_test9", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + }; + + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 and s1 < 19.0 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "1,d1,1.0,true,aligned_test1", + "2,d1,2.0,null,aligned_test2", + "4,d1,4.0,true,aligned_test4", + "5,d1,5.0,true,aligned_test5", + "6,d1,6.0,true,null", + "7,d1,7.0,false,aligned_test7", + "8,d1,8.0,null,aligned_test8", + "9,d1,9.0,false,aligned_test9", + "11,d1,11.0,null,null", + "12,d1,12.0,null,null", + "14,d1,14.0,null,null", + "15,d1,15.0,null,null", + "16,d1,16.0,null,null", + "34,d1,null,null,aligned_test34", + }; + + String[] columnNames = {"Device", "s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s4,s5 from table0 where device='d1' and s1 < 17.0 or s5 = 'aligned_test34' order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterAlignByDeviceTest2() { + String[] retArray = + new String[] { + "7,d1,7.0,false", "9,d1,9.0,false", + }; + + String[] columnNames = {"Device", "s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s4 from table0 where device='d1' and s1 < 19.0 and s4 = false order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithoutTimeFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "20", "29", "28", "19", "20"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedAndNonAlignedWithoutTimeFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1,20,29,28,19,20,", "d2,19,29,28,18,19,"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from root.sg1.* align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(resultSet.getString(index)).append(","); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithTimeFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "12", "15", "22", "13", "6"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(*) from d1 where time >= 9 and time <= 33 align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** aggregate multi columns of aligned timeseries in one SQL */ + @Ignore + @Test + public void aggregateSomeAlignedWithoutTimeFilterAlignByDeviceTest() { + double[] retArray = + new double[] { + 20, 29, 28, 390184, 130549, 390417, 19509.2, 4501.689655172413, 13943.464285714286 + }; + String[] columnNames = { + "Device", + "count(s1)", + "count(s2)", + "count(s3)", + "sum(s1)", + "sum(s2)", + "sum(s3)", + "avg(s1)", + "avg(s2)", + "avg(s3)", + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1),count(s2),count(s3),sum(s1),sum(s2),sum(s3),avg(s1),avg(s2),avg(s3) from d1 align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 1; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i - 1] = Double.parseDouble(resultSet.getString(index)); + assertEquals(retArray[i - 1], ans[i - 1], DELTA); + } + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "11"}; + String[] columnNames = {"Device", "count(s4)"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(s4) from d1 where s4 = true align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationFuncAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = + new String[] {"d1", "8", "42.0", "5.25", "1.0", "9.0", "1", "9", "1.0", "9.0"}; + String[] columnNames = { + "Device", + "count(s1)", + "sum(s1)", + "avg(s1)", + "first_value(s1)", + "last_value(s1)", + "min_time(s1)", + "max_time(s1)", + "min_value(s1)", + "max_value(s1)", + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s1), avg(s1), " + + "first_value(s1), last_value(s1), " + + "min_time(s1), max_time(s1)," + + "max_value(s1), min_value(s1) from d1 where s1 < 10 " + + "align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "6", "6", "9", "11", "6"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 where s4 = true " + "align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationAllAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "160016.0", "11", "1", "13"}; + String[] columnNames = { + "Device", "sum(s1)", "count(s4)", "min_value(s3)", "max_time(s2)", + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select sum(s1), count(s4), min_value(s3), max_time(s2) from d1 where s4 = true " + + "align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countSumAvgGroupByTimeAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,4,40.0,7.5", + "11,d1,10,130142.0,13014.2", + "21,d1,1,null,230000.0", + "31,d1,0,355.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueGroupByTimeAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,10,6.0,10,6", + "11,d1,130000,11.0,20,11", + "21,d1,230000,230000.0,null,21", + "31,d1,null,null,40,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(maxValue("s3")) + + "," + + resultSet.getString(minValue("s1")) + + "," + + resultSet.getString(maxTime("s2")) + + "," + + resultSet.getString(minTime("s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByTimeAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,null,null", "6,d1,true,aligned_test7", + "11,d1,true,aligned_unseq_test13", "16,d1,null,null", + "21,d1,true,null", "26,d1,false,null", + "31,d1,null,aligned_test31", "36,d1,null,aligned_test36" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(lastValue("s4")) + + "," + + resultSet.getString(firstValue("s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,9,9,8,8,9,9.0,10,10,true,aligned_test10", + "11,d1,10,10,10,1,1,20.0,20,20,true,aligned_unseq_test13", + "21,d1,1,0,10,10,0,230000.0,null,30,false,null", + "31,d1,0,10,0,0,10,null,40,null,null,aligned_test40" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(*), last_value(*) from d1 GROUP BY ([1, 41), 10ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(count("s2")) + + "," + + resultSet.getString(count("s3")) + + "," + + resultSet.getString(count("s4")) + + "," + + resultSet.getString(count("s5")) + + "," + + resultSet.getString(lastValue("s1")) + + "," + + resultSet.getString(lastValue("s2")) + + "," + + resultSet.getString(lastValue("s3")) + + "," + + resultSet.getString(lastValue("s4")) + + "," + + resultSet.getString(lastValue("s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithNonAlignedTimeseriesAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,0,null,null", + "7,d1,3,34.0,8.0", + "13,d1,4,130045.0,32511.25", + "19,d1,2,39.0,19.5", + "25,d1,0,null,null", + "31,d1,0,130.0,null", + "37,d1,0,154.0,null", + "1,d2,0,null,null", + "7,d2,3,34.0,8.0", + "13,d2,4,58.0,14.5", + "19,d2,2,39.0,19.5", + "25,d2,0,null,null", + "31,d2,0,130.0,null", + "37,d2,0,154.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from root.sg1.* " + + "where time > 5 GROUP BY ([1, 41), 4ms, 6ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgPreviousFillAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,4,40.0,7.5", + "11,d1,10,130142.0,13014.2", + "21,d1,1,130142.0,230000.0", + "31,d1,0,355.0,230000.0" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms) FILL (previous) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgValueFillAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,1,3.0,30000.0", + "6,d1,4,40.0,7.5", + "11,d1,5,130052.0,26010.4", + "16,d1,5,90.0,18.0", + "21,d1,1,3.0,230000.0", + "26,d1,0,3.0,3.0", + "31,d1,0,3.0,3.0" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where s3 > 5 and time < 30 GROUP BY ([1, 36), 5ms) FILL (3) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + // --------------------------GroupByWithoutValueFilter-------------------------- + @Ignore + @Test + public void countSumAvgGroupByTest1() throws SQLException { + String[] retArray = + new String[] { + "1,4,40.0,7.5", "11,10,130142.0,13014.2", "21,1,null,230000.0", "31,0,355.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + " where time > 5 GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgGroupByTest2() throws SQLException { + String[] retArray = + new String[] { + "1,0,null,null", "6,4,40.0,7.5", "11,5,130052.0,26010.4", "16,5,90.0,18.0", + "21,1,null,230000.0", "26,0,null,null", "31,0,165.0,null", "36,0,73.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgGroupByWithSlidingStepTest() throws SQLException { + String[] retArray = + new String[] { + "1,0,null,null", + "7,3,34.0,8.0", + "13,4,130045.0,32511.25", + "19,2,39.0,19.5", + "25,0,null,null", + "31,0,130.0,null", + "37,0,154.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + " where time > 5 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgGroupByWithNonAlignedTimeseriesTest() throws SQLException { + String[] retArray = + new String[] { + "1,0,null,null,0,null,null", + "7,3,34.0,8.0,4,34.0,8.5", + "13,4,58.0,14.5,4,130045.0,14.5", + "19,2,39.0,19.5,4,39.0,20.5", + "25,0,null,null,4,null,26.5", + "31,0,130.0,null,0,130.0,null", + "37,0,154.0,null,0,154.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(d1.s1), sum(d2.s2), avg(d2.s1), count(d1.s3), sum(d1.s2), avg(d2.s3) " + + "from root.sg1 where time > 5 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d2.s2")) + + "," + + resultSet.getString(avg("d2.s1")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d2.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(d1.s1), sum(d2.s2), avg(d2.s1), count(d1.s3), sum(d1.s2), avg(d2.s3) " + + "from root.sg1 where time > 5 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d2.s2")) + + "," + + resultSet.getString(avg("d2.s1")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d2.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByTest1() throws SQLException { + String[] retArray = + new String[] { + "1,10,6.0,10,6", + "11,130000,11.0,20,11", + "21,230000,230000.0,null,21", + "31,null,null,40,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + " where time > 5 GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByTest2() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null", + "6,10,6.0,10,6", + "11,130000,11.0,15,11", + "16,20,16.0,20,16", + "21,230000,230000.0,null,21", + "26,30,null,null,26", + "31,null,null,35,null", + "36,null,null,37,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByWithSlidingStepTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null", + "7,10,7.0,10,7", + "13,130000,14.0,16,13", + "19,22,19.0,20,19", + "25,28,null,null,25", + "31,null,null,34,null", + "37,null,null,40,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + " where time > 5 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByWithNonAlignedTimeseriesTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null,null,null,null,null", + "7,10,7.0,10,7,10,7.0,10,7", + "13,16,14.0,16,13,130000,13.0,16,13", + "19,22,19.0,20,19,22,19.0,20,19", + "25,28,null,null,25,28,null,null,25", + "31,null,null,34,null,null,null,34,null", + "37,null,null,37,null,null,null,37,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(d2.s3), min_value(d1.s1), max_time(d2.s2), min_time(d1.s3), " + + "max_value(d1.s3), min_value(d2.s1), max_time(d1.s2), min_time(d2.s3) " + + "from root.sg1 where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d2.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d2.s2")) + + "," + + resultSet.getString(minTime("d1.s3")) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d2.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d2.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(d2.s3), min_value(d1.s1), max_time(d2.s2), min_time(d1.s3), " + + "max_value(d1.s3), min_value(d2.s1), max_time(d1.s2), min_time(d2.s3) " + + "from root.sg1 where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d2.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d2.s2")) + + "," + + resultSet.getString(minTime("d1.s3")) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d2.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d2.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByTest1() throws SQLException { + String[] retArray = + new String[] { + "1,true,aligned_test7", + "11,true,aligned_unseq_test13", + "21,false,null", + "31,null,aligned_test31" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + " where time > 5 GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByTest2() throws SQLException { + String[] retArray = + new String[] { + "1,null,null", "6,true,aligned_test7", "11,true,aligned_unseq_test13", "16,null,null", + "21,true,null", "26,false,null", "31,null,aligned_test31", "36,null,aligned_test36" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByWithSlidingStepTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null", + "7,true,aligned_test7", + "13,true,aligned_unseq_test13", + "19,true,null", + "25,false,null", + "31,null,aligned_test31", + "37,null,aligned_test37" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByWithNonAlignedTimeseriesTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null", + "7,non_aligned_test10,false,aligned_test10,false", + "13,null,true,aligned_unseq_test13,null", + "19,null,true,null,true", + "25,null,true,null,true", + "31,non_aligned_test34,null,aligned_test34,null", + "37,non_aligned_test37,null,aligned_test37,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(d2.s5), first_value(d1.s4), last_value(d1.s5), first_value(d2.s4) " + + "from root.sg1 where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d2.s5")) + + "," + + resultSet.getString(firstValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")) + + "," + + resultSet.getString(firstValue("d2.s4")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(d2.s5), first_value(d1.s4), last_value(d1.s5), first_value(d2.s4) " + + "from root.sg1 where time > 5 and time < 38 " + + "GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d2.s5")) + + "," + + resultSet.getString(firstValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")) + + "," + + resultSet.getString(firstValue("d2.s4")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardTest1() throws SQLException { + String[] retArray = + new String[] { + "1,9,9,8,8,9,9.0,10,10,true,aligned_test10", + "11,10,10,10,1,1,20.0,20,20,true,aligned_unseq_test13", + "21,1,0,10,10,0,230000.0,null,30,false,null", + "31,0,10,0,0,10,null,40,null,null,aligned_test40" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(*), last_value(*) from d1 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(*), last_value(*) from d1 " + + "GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardTest2() throws SQLException { + String[] retArray = + new String[] { + "1,0,0,0,0,0", + "5,2,2,2,2,1", + "9,2,3,3,2,2", + "13,3,3,3,1,1", + "17,3,3,3,0,0", + "21,1,0,3,3,0", + "25,0,0,3,3,0", + "29,0,1,2,2,1", + "33,0,3,0,0,3", + "37,0,1,0,0,1" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(*) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(*) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardTest3() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null,null", + "5,7.0,7,7,false,aligned_test7", + "9,11.0,11,11,true,aligned_test10", + "13,15.0,15,15,true,aligned_unseq_test13", + "17,19.0,19,19,null,null", + "21,230000.0,null,230000,false,null", + "25,null,null,27,false,null", + "29,null,31,30,false,aligned_test31", + "33,null,35,null,null,aligned_test35", + "37,null,37,null,null,aligned_test37" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(*) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(*) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithoutAggregationFuncTest() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + statement.executeQuery("select s1 from d1 group by ([0, 100), 5ms)"); + + fail("No expected exception thrown"); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage(), + e.getMessage() + .contains( + "Common queries and aggregated queries are not allowed to appear at the same time")); + } + } + + @Ignore + @Test + public void negativeOrZeroTimeIntervalTest() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 0ms)"); + fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage(), + e.getMessage() + .contains("The second parameter time interval should be a positive integer.")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable2IT.java new file mode 100644 index 0000000000000..bf16fe3bd4033 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable2IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTable2IT extends IoTDBPredicatePushDownTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(2); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable3IT.java new file mode 100644 index 0000000000000..ef8d81479bb5d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable3IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTable3IT extends IoTDBPredicatePushDownTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable4IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable4IT.java new file mode 100644 index 0000000000000..e2f215f3a978e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTable4IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTable4IT extends IoTDBPredicatePushDownTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTableIT.java new file mode 100644 index 0000000000000..7b4f933dfd778 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/IoTDBPredicatePushDownTableIT.java @@ -0,0 +1,758 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTableIT { + + private final String database = "db"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAlignedRawDataAlignByTime1() { + String[] expectedHeader1 = new String[] {"Time", "s2", "s3"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.010Z,10,10,", + "1970-01-01T00:00:00.011Z,11,11,", + "1970-01-01T00:00:00.012Z,12,12,", + "1970-01-01T00:00:00.014Z,14,14,", + "1970-01-01T00:00:00.015Z,15,15,", + "1970-01-01T00:00:00.016Z,16,16,", + "1970-01-01T00:00:00.017Z,17,17,", + "1970-01-01T00:00:00.018Z,18,18,", + "1970-01-01T00:00:00.019Z,19,19,", + "1970-01-01T00:00:00.020Z,20,20," + }; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30", + expectedHeader1, + retArray1, + database); + + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d1' and 9 <= s2 - 1 and 30 > s2", + expectedHeader1, + retArray1, + database); + + retArray1 = new String[] {"1970-01-01T00:00:00.020Z,20,20,"}; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d1' and 9 <= s2 - 1 and 30 > s2 and 19 < time", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s3"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s2"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s2"}; + String[] retArray4 = + new String[] {"1970-01-01T00:00:00.014Z,14,", "1970-01-01T00:00:00.015Z,15,"}; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30 offset 3 limit 2", + expectedHeader4, + retArray4, + database); + } + + // TODO fix it + @Test + public void testAlignedRawDataAlignByTime2() { + String[] expectedHeader1 = new String[] {"Time", "s2", "_col2"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.003Z,null,30001,", + "1970-01-01T00:00:00.013Z,130000,130001,", + "1970-01-01T00:00:00.016Z,16,17,", + "1970-01-01T00:00:00.017Z,17,18,", + "1970-01-01T00:00:00.018Z,18,19,", + "1970-01-01T00:00:00.019Z,19,20,", + "1970-01-01T00:00:00.020Z,20,21,", + "1970-01-01T00:00:00.021Z,null,22,", + "1970-01-01T00:00:00.022Z,null,23,", + "1970-01-01T00:00:00.023Z,null,230001,", + "1970-01-01T00:00:00.024Z,null,25,", + "1970-01-01T00:00:00.025Z,null,26,", + "1970-01-01T00:00:00.026Z,null,27,", + "1970-01-01T00:00:00.027Z,null,28,", + "1970-01-01T00:00:00.028Z,null,29,", + "1970-01-01T00:00:00.029Z,null,30,", + "1970-01-01T00:00:00.030Z,null,31,", + }; + tableResultSetEqualTest( + "select Time,s2, s3 + 1 from table0 where device='d1' and s3 + 1 > 16", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s2"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.003Z,null,", + "1970-01-01T00:00:00.013Z,130000,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,null,", + "1970-01-01T00:00:00.022Z,null,", + "1970-01-01T00:00:00.023Z,null,", + "1970-01-01T00:00:00.024Z,null,", + "1970-01-01T00:00:00.025Z,null,", + "1970-01-01T00:00:00.026Z,null,", + "1970-01-01T00:00:00.027Z,null,", + "1970-01-01T00:00:00.028Z,null,", + "1970-01-01T00:00:00.029Z,null,", + "1970-01-01T00:00:00.030Z,null,", + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d1' and s3 + 1 > 16", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s3"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.003Z,30000,", + "1970-01-01T00:00:00.013Z,130000,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,21,", + "1970-01-01T00:00:00.022Z,22,", + "1970-01-01T00:00:00.023Z,230000,", + "1970-01-01T00:00:00.024Z,24,", + "1970-01-01T00:00:00.025Z,25,", + "1970-01-01T00:00:00.026Z,26,", + "1970-01-01T00:00:00.027Z,27,", + "1970-01-01T00:00:00.028Z,28,", + "1970-01-01T00:00:00.029Z,29,", + "1970-01-01T00:00:00.030Z,30,", + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d1' and s3 + 1 > 16", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s3"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.003Z,30000,", + "1970-01-01T00:00:00.013Z,130000,", + "1970-01-01T00:00:00.016Z,16," + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d1' and s3 + 1 > 16 limit 3", + expectedHeader4, + retArray4, + database); + } + + @Test + public void testNonAlignedRawDataAlignByTime1() { + String[] expectedHeader1 = new String[] {"Time", "s2", "s3"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.010Z,10,10,", + "1970-01-01T00:00:00.011Z,11,11,", + "1970-01-01T00:00:00.012Z,12,12,", + "1970-01-01T00:00:00.013Z,13,13,", + "1970-01-01T00:00:00.014Z,14,14,", + "1970-01-01T00:00:00.015Z,15,15,", + "1970-01-01T00:00:00.016Z,16,16,", + "1970-01-01T00:00:00.017Z,17,17,", + "1970-01-01T00:00:00.018Z,18,18,", + "1970-01-01T00:00:00.019Z,19,19,", + "1970-01-01T00:00:00.020Z,20,20," + }; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s3"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.013Z,13,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s2"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.013Z,13,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s2"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.013Z,13,", + "1970-01-01T00:00:00.014Z,14," + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30 offset 2 limit 3", + expectedHeader4, + retArray4, + database); + } + + @Test + public void testNonAlignedRawDataAlignByTime2() { + String[] expectedHeader1 = new String[] {"Time", "s2", "s3"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.016Z,16,16,", + "1970-01-01T00:00:00.017Z,17,17,", + "1970-01-01T00:00:00.018Z,18,18,", + "1970-01-01T00:00:00.019Z,19,19,", + "1970-01-01T00:00:00.020Z,20,20,", + "1970-01-01T00:00:00.021Z,null,21,", + "1970-01-01T00:00:00.022Z,null,22,", + "1970-01-01T00:00:00.023Z,null,23,", + "1970-01-01T00:00:00.024Z,null,24,", + "1970-01-01T00:00:00.025Z,null,25,", + "1970-01-01T00:00:00.026Z,null,26,", + "1970-01-01T00:00:00.027Z,null,27,", + "1970-01-01T00:00:00.028Z,null,28,", + "1970-01-01T00:00:00.029Z,null,29,", + "1970-01-01T00:00:00.030Z,null,30,", + }; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d2' and s3 + 1 > 16", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s2"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,null,", + "1970-01-01T00:00:00.022Z,null,", + "1970-01-01T00:00:00.023Z,null,", + "1970-01-01T00:00:00.024Z,null,", + "1970-01-01T00:00:00.025Z,null,", + "1970-01-01T00:00:00.026Z,null,", + "1970-01-01T00:00:00.027Z,null,", + "1970-01-01T00:00:00.028Z,null,", + "1970-01-01T00:00:00.029Z,null,", + "1970-01-01T00:00:00.030Z,null,", + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d2' and s3 + 1 > 16", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s3"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,21,", + "1970-01-01T00:00:00.022Z,22,", + "1970-01-01T00:00:00.023Z,23,", + "1970-01-01T00:00:00.024Z,24,", + "1970-01-01T00:00:00.025Z,25,", + "1970-01-01T00:00:00.026Z,26,", + "1970-01-01T00:00:00.027Z,27,", + "1970-01-01T00:00:00.028Z,28,", + "1970-01-01T00:00:00.029Z,29,", + "1970-01-01T00:00:00.030Z,30,", + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d2' and s3 + 1 > 16", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s3"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.026Z,26,", + "1970-01-01T00:00:00.027Z,27,", + "1970-01-01T00:00:00.028Z,28,", + "1970-01-01T00:00:00.029Z,29,", + "1970-01-01T00:00:00.030Z,30,", + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d2' and s3 + 1 > 16 offset 10", + expectedHeader4, + retArray4, + database); + } + + @Ignore + @Test + public void testAlignedAggregationAlignByTime1() { + String[] expectedHeader1 = new String[] {"count(d1.s2),count(d1.s3),"}; + String[] retArray1 = + new String[] { + "10,10,", + }; + tableResultSetEqualTest( + "select count(s2), count(s3) from d1 where s2 - 1 >= 9 and s2 < 30", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"count(d1.s3),"}; + String[] retArray2 = new String[] {"10,"}; + tableResultSetEqualTest( + "select count(s3) from d1 where s2 - 1 >= 9 and s2 < 30", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"count(d1.s2),"}; + String[] retArray3 = + new String[] { + "10,", + }; + tableResultSetEqualTest( + "select count(s2) from d1 where s2 - 1 >= 9 and s2 < 30", + expectedHeader3, + retArray3, + database); + } + + // @Ignore + // @Test + // public void testAlignedAggregationAlignByTime2() { + // String[] expectedHeader1 = new String[]{"count(d1.s2),count(d1.s3 + 1),"}; + // String[] retArray1 = + // new String[] { + // "6,17,", + // }; + // tableResultSetEqualTest( + // "select count(s2), count(s3 + 1) from d1 where s3 + 1 > 16", + // expectedHeader1, + // retArray1, database); + // + // String[] expectedHeader2 = "count(d1.s2),"; + // String[] retArray2 = + // new String[] { + // "6,", + // }; + // tableResultSetEqualTest( + // "select count(s2) from d1 where s3 + 1 > 16", expectedHeader2, retArray2, database); + // + // String[] expectedHeader3 = "count(d1.s3),"; + // String[] retArray3 = + // new String[] { + // "17,", + // }; + // tableResultSetEqualTest( + // "select count(s3) from d1 where s3 + 1 > 16", expectedHeader3, retArray3, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedAggregationAlignByTime1() { + // String[] expectedHeader1 = "count(d2.s2),count(d2.s3),"; + // String[] retArray1 = new String[] {"11,11,"}; + // tableResultSetEqualTest( + // "select count(s2), count(s3) from d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader1, + // retArray1, database); + // + // String[] expectedHeader2 = "count(d2.s3),"; + // String[] retArray2 = new String[] {"11,"}; + // tableResultSetEqualTest( + // "select count(s3) from d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader2, + // retArray2, database); + // + // String[] expectedHeader3 = "count(d2.s2),"; + // String[] retArray3 = new String[] {"11,"}; + // tableResultSetEqualTest( + // "select count(s2) from d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader3, + // retArray3, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedAggregationAlignByTime2() { + // String[] expectedHeader1 = "count(d2.s2),count(d2.s3),"; + // String[] retArray1 = + // new String[] { + // "5,15,", + // }; + // tableResultSetEqualTest( + // "select count(s2), count(s3) from d2 where s3 + 1 > 16", + // expectedHeader1, + // retArray1, database); + // + // String[] expectedHeader2 = "count(d2.s2),"; + // String[] retArray2 = + // new String[] { + // "5,", + // }; + // tableResultSetEqualTest( + // "select count(s2) from d2 where s3 + 1 > 16", expectedHeader2, retArray2, database); + // + // String[] expectedHeader3 = "count(d2.s3),"; + // String[] retArray3 = + // new String[] { + // "15,", + // }; + // tableResultSetEqualTest( + // "select count(s3) from d2 where s3 + 1 > 16", expectedHeader3, retArray3, database); + // } + // + // @Ignore + // @Test + // public void testMixAggregationAlignByTime() { + // String[] expectedHeader1 = + // "count(d1.s2),count(d2.s2),count(d1.s3),count(d2.s3),"; + // String[] retArray1 = + // new String[] { + // "10,10,10,10,", + // }; + // tableResultSetEqualTest( + // "select count(s2), count(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader1, + // retArray1, database); + // } + // + // @Ignore + // @Test + // public void testAlignedGroupByTimeAlignByTime1() { + // String[] expectedHeader = "Time,count(d1.s2),sum(d1.s3),"; + // String[] retArray = new String[] {"1,1,10.0,", "11,9,142.0,", "21,0,null,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d1 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), + // 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testAlignedGroupByTimeAlignByTime2() { + // String[] expectedHeader = "Time,count(d1.s2),sum(d1.s3),"; + // String[] retArray = + // new String[] {"1,0,30000.0,", "11,6,130090.0,", "21,0,230232.0,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d1 where s3 + 1 > 16 group by ([1, 41), 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedGroupByTimeAlignByTime1() { + // String[] expectedHeader = "Time,count(d2.s2),sum(d2.s3 + 1),"; + // String[] retArray = new String[] {"1,1,11.0,", "11,10,165.0,", "21,0,null,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3 + 1) from d2 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), + // 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedGroupByTimeAlignByTime2() { + // String[] expectedHeader = "Time,count(d2.s2),sum(d2.s3),"; + // String[] retArray = new String[] {"1,0,null,", "11,5,90.0,", "21,0,255.0,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d2 where s3 + 1 > 16 group by ([1, 41), 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testMixGroupByTimeAlignByTime() { + // String[] expectedHeader = + // "Time,count(d1.s2),count(d2.s2),sum(d1.s3),sum(d2.s3),"; + // String[] retArray = + // new String[] { + // "1,1,1,10.0,10.0,", "11,9,9,142.0,142.0,", "21,0,0,null,null,", "31,0,0,null,null," + // }; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), + // 10ms)", + // expectedHeader, + // retArray, database); + // } + + @Test + public void testRawDataAlignByDevice1() { + String[] expectedHeader = new String[] {"Time", "Device", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.010Z,d2,10,10,", + "1970-01-01T00:00:00.011Z,d2,11,11,", + "1970-01-01T00:00:00.012Z,d2,12,12,", + "1970-01-01T00:00:00.013Z,d2,13,13,", + "1970-01-01T00:00:00.014Z,d2,14,14,", + "1970-01-01T00:00:00.015Z,d2,15,15,", + "1970-01-01T00:00:00.016Z,d2,16,16,", + "1970-01-01T00:00:00.017Z,d2,17,17,", + "1970-01-01T00:00:00.018Z,d2,18,18,", + "1970-01-01T00:00:00.019Z,d2,19,19,", + "1970-01-01T00:00:00.020Z,d2,20,20,", + "1970-01-01T00:00:00.010Z,d1,10,10,", + "1970-01-01T00:00:00.011Z,d1,11,11,", + "1970-01-01T00:00:00.012Z,d1,12,12,", + "1970-01-01T00:00:00.014Z,d1,14,14,", + "1970-01-01T00:00:00.015Z,d1,15,15,", + "1970-01-01T00:00:00.016Z,d1,16,16,", + "1970-01-01T00:00:00.017Z,d1,17,17,", + "1970-01-01T00:00:00.018Z,d1,18,18,", + "1970-01-01T00:00:00.019Z,d1,19,19,", + "1970-01-01T00:00:00.020Z,d1,20,20,", + }; + tableResultSetEqualTest( + "select Time, Device,s2, s3 from table0 where s2 - 1 >= 9 and s2 < 30 order by device desc", + expectedHeader, + retArray, + database); + } + + @Test + public void testRawDataAlignByDevice2() { + String[] expectedHeader = new String[] {"Time", "Device", "s2", "_col3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,30001,", + "1970-01-01T00:00:00.013Z,d1,130000,130001,", + "1970-01-01T00:00:00.016Z,d1,16,17,", + "1970-01-01T00:00:00.017Z,d1,17,18,", + "1970-01-01T00:00:00.018Z,d1,18,19,", + "1970-01-01T00:00:00.019Z,d1,19,20,", + "1970-01-01T00:00:00.020Z,d1,20,21,", + "1970-01-01T00:00:00.021Z,d1,null,22,", + "1970-01-01T00:00:00.022Z,d1,null,23,", + "1970-01-01T00:00:00.023Z,d1,null,230001,", + "1970-01-01T00:00:00.024Z,d1,null,25,", + "1970-01-01T00:00:00.025Z,d1,null,26,", + "1970-01-01T00:00:00.026Z,d1,null,27,", + "1970-01-01T00:00:00.027Z,d1,null,28,", + "1970-01-01T00:00:00.028Z,d1,null,29,", + "1970-01-01T00:00:00.029Z,d1,null,30,", + "1970-01-01T00:00:00.030Z,d1,null,31,", + "1970-01-01T00:00:00.016Z,d2,16,17,", + "1970-01-01T00:00:00.017Z,d2,17,18,", + "1970-01-01T00:00:00.018Z,d2,18,19,", + "1970-01-01T00:00:00.019Z,d2,19,20,", + "1970-01-01T00:00:00.020Z,d2,20,21,", + "1970-01-01T00:00:00.021Z,d2,null,22,", + "1970-01-01T00:00:00.022Z,d2,null,23,", + "1970-01-01T00:00:00.023Z,d2,null,24,", + "1970-01-01T00:00:00.024Z,d2,null,25,", + "1970-01-01T00:00:00.025Z,d2,null,26,", + "1970-01-01T00:00:00.026Z,d2,null,27,", + "1970-01-01T00:00:00.027Z,d2,null,28,", + "1970-01-01T00:00:00.028Z,d2,null,29,", + "1970-01-01T00:00:00.029Z,d2,null,30,", + "1970-01-01T00:00:00.030Z,d2,null,31,", + }; + tableResultSetEqualTest( + "select Time,Device,s2, s3 + 1 from table0 where s3 + 1 > 16 order by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testAggregationAlignByDevice1() { + String[] expectedHeader = new String[] {"Device,count(s2),sum(s3),"}; + String[] retArray = new String[] {"d2,11,165.0,", "d1,10,152.0,"}; + tableResultSetEqualTest( + "select count(s2), sum(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30 order by device desc align by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testAggregationAlignByDevice2() { + String[] expectedHeader = new String[] {"Device,count(s2),sum(s3 + 1),"}; + String[] retArray = new String[] {"d1,6,390339.0,", "d2,5,360.0,"}; + tableResultSetEqualTest( + "select count(s2), sum(s3 + 1) from d1, d2 where s3 + 1 > 16 align by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testGroupByTimeAlignByDevice1() { + String[] expectedHeader = new String[] {"Time,Device,count(s2),sum(s3),"}; + String[] retArray = + new String[] { + "1,d2,1,10.0,", + "11,d2,10,155.0,", + "21,d2,0,null,", + "31,d2,0,null,", + "1,d1,1,10.0,", + "11,d1,9,142.0,", + "21,d1,0,null,", + "31,d1,0,null," + }; + tableResultSetEqualTest( + "select count(s2), sum(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), 10ms) order by device desc align by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testGroupByTimeAlignByDevice2() { + String[] expectedHeader = new String[] {"Time,Device,count(s2),sum(s3 + 1),"}; + String[] retArray = + new String[] { + "1,d1,0,30001.0,", + "11,d1,6,130096.0,", + "21,d1,0,230242.0,", + "31,d1,0,null,", + "1,d2,0,null,", + "11,d2,5,95.0,", + "21,d2,0,265.0,", + "31,d2,0,null," + }; + tableResultSetEqualTest( + "select count(s2), sum(s3 + 1) from d1, d2 where s3 + 1 > 16 group by ([1, 41), 10ms) align by device", + expectedHeader, + retArray, + database); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/TableUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/TableUtils.java new file mode 100644 index 0000000000000..71f936a0d3b8c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/aligned/TableUtils.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.aligned; + +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * This class generates data for test cases in aligned time series scenarios. + * + *

You can comprehensively view the generated data in the following online doc: + * + *

https://docs.google.com/spreadsheets/d/1kfrSR1_paSd9B1Z0jnPBD3WQIMDslDuNm4R0mpWx9Ms/edit?usp=sharing + */ +public class TableUtils { + + public static final String USE_DB = "use db"; + + private static final String[] sqls = + new String[] { + "CREATE DATABASE db", + "USE db", + "CREATE TABLE table0 (device string tag, s1 FLOAT field, s2 INT32 field, s3 INT64 field, s4 BOOLEAN field, s5 TEXT field)", + "insert into table0(device, time, s1, s2, s3, s4, s5) values('d1', 1, 1.0, 1, 1, TRUE, 'aligned_test1')", + "insert into table0(device, time, s1, s2, s3, s5) values('d1', 2, 2.0, 2, 2, 'aligned_test2')", + "insert into table0(device, time, s1, s3, s4, s5) values('d1', 3, 3.0, 3, FALSE, 'aligned_test3')", + "insert into table0(device, time, s1, s2, s4, s5) values('d1', 4, 4.0, 4, TRUE, 'aligned_test4')", + "insert into table0(device, time, s1, s2, s4, s5) values('d1', 5, 5.0, 5, TRUE, 'aligned_test5')", + "insert into table0(device, time, s1, s2, s3, s4) values('d1', 6, 6.0, 6, 6, TRUE)", + "insert into table0(device, time, s1, s2, s3, s4, s5) values('d1',7, 7.0, 7, 7, FALSE, 'aligned_test7')", + "insert into table0(device, time, s1, s2, s3, s5) values('d1',8, 8.0, 8, 8, 'aligned_test8')", + "insert into table0(device, time, s1, s2, s3, s4, s5) values('d1',9, 9.0, 9, 9, FALSE, 'aligned_test9')", + "insert into table0(device, time, s2, s3, s4, s5) values('d1',10, 10, 10, TRUE, 'aligned_test10')", + "insert into table0(device, time, s1, s2, s3, s4, s5) values('d2',1, 1.0, 1, 1, TRUE, 'non_aligned_test1')", + "insert into table0(device, time, s1, s2, s3, s5) values('d2',2, 2.0, 2, 2, 'non_aligned_test2')", + "insert into table0(device, time, s1, s3, s4, s5) values('d2',3, 3.0, 3, FALSE, 'non_aligned_test3')", + "insert into table0(device, time, s1, s2, s4, s5) values('d2',4, 4.0, 4, TRUE, 'non_aligned_test4')", + "insert into table0(device, time, s1, s2, s4, s5) values('d2',5, 5.0, 5, TRUE, 'non_aligned_test5')", + "insert into table0(device, time, s1, s2, s3, s4) values('d2',6, 6.0, 6, 6, TRUE)", + "insert into table0(device, time, s1, s2, s3, s4, s5) values('d2',7, 7.0, 7, 7, FALSE, 'non_aligned_test7')", + "insert into table0(device, time, s1, s2, s3, s5) values('d2',8, 8.0, 8, 8, 'non_aligned_test8')", + "insert into table0(device, time, s1, s2, s3, s4, s5) values('d2',9, 9.0, 9, 9, FALSE, 'non_aligned_test9')", + "insert into table0(device, time, s2, s3, s4, s5) values('d2',10, 10, 10, TRUE, 'non_aligned_test10')", + "flush", + "insert into table0(device,time, s1, s3, s4, s5) values('d1',3, 30000.0, 30000, TRUE, 'aligned_unseq_test3')", + "insert into table0(device,time, s1, s2, s3) values('d1',11, 11.0, 11, 11)", + "insert into table0(device,time, s1, s2, s3) values('d1',12, 12.0, 12, 12)", + "insert into table0(device,time, s1, s2, s3) values('d1',13, 13.0, 13, 13)", + "insert into table0(device,time, s1, s2, s3) values('d1',14, 14.0, 14, 14)", + "insert into table0(device,time, s1, s2, s3) values('d1',15, 15.0, 15, 15)", + "insert into table0(device,time, s1, s2, s3) values('d1',16, 16.0, 16, 16)", + "insert into table0(device,time, s1, s2, s3) values('d1',17, 17.0, 17, 17)", + "insert into table0(device,time, s1, s2, s3) values('d1',18, 18.0, 18, 18)", + "insert into table0(device,time, s1, s2, s3) values('d1',19, 19.0, 19, 19)", + "insert into table0(device,time, s1, s2, s3) values('d1',20, 20.0, 20, 20)", + "insert into table0(device,time, s1, s2, s3) values('d2',11, 11.0, 11, 11)", + "insert into table0(device,time, s1, s2, s3) values('d2',12, 12.0, 12, 12)", + "insert into table0(device,time, s1, s2, s3) values('d2',13, 13.0, 13, 13)", + "insert into table0(device,time, s1, s2, s3) values('d2',14, 14.0, 14, 14)", + "insert into table0(device,time, s1, s2, s3) values('d2',15, 15.0, 15, 15)", + "insert into table0(device,time, s1, s2, s3) values('d2',16, 16.0, 16, 16)", + "insert into table0(device,time, s1, s2, s3) values('d2',17, 17.0, 17, 17)", + "insert into table0(device,time, s1, s2, s3) values('d2',18, 18.0, 18, 18)", + "insert into table0(device,time, s1, s2, s3) values('d2',19, 19.0, 19, 19)", + "insert into table0(device,time, s1, s2, s3) values('d2',20, 20.0, 20, 20)", + "flush", + "insert into table0(device,time, s1, s2, s3, s4, s5) values('d1',13, 130000.0, 130000, 130000, TRUE, 'aligned_unseq_test13')", + "insert into table0(device,time, s3, s4) values('d1',21, 21, TRUE)", + "insert into table0(device,time, s3, s4) values('d1',22, 22, TRUE)", + "insert into table0(device,time, s3, s4) values('d1',23, 23, TRUE)", + "insert into table0(device,time, s3, s4) values('d1',24, 24, TRUE)", + "insert into table0(device,time, s3, s4) values('d1',25, 25, TRUE)", + "insert into table0(device,time, s3, s4) values('d1',26, 26, FALSE)", + "insert into table0(device,time, s3, s4) values('d1',27, 27, FALSE)", + "insert into table0(device,time, s3, s4) values('d1',28, 28, FALSE)", + "insert into table0(device,time, s3, s4) values('d1',29, 29, FALSE)", + "insert into table0(device,time, s3, s4) values('d1',30, 30, FALSE)", + "insert into table0(device,time, s3, s4) values('d2',21, 21, TRUE)", + "insert into table0(device,time, s3, s4) values('d2',22, 22, TRUE)", + "insert into table0(device,time, s3, s4) values('d2',23, 23, TRUE)", + "insert into table0(device,time, s3, s4) values('d2',24, 24, TRUE)", + "insert into table0(device,time, s3, s4) values('d2',25, 25, TRUE)", + "insert into table0(device,time, s3, s4) values('d2',26, 26, FALSE)", + "insert into table0(device,time, s3, s4) values('d2',27, 27, FALSE)", + "insert into table0(device,time, s3, s4) values('d2',28, 28, FALSE)", + "insert into table0(device,time, s3, s4) values('d2',29, 29, FALSE)", + "insert into table0(device,time, s3, s4) values('d2',30, 30, FALSE)", + "flush", + "insert into table0(device,time, s1, s3, s4) values('d1',23, 230000.0, 230000, FALSE)", + "insert into table0(device,time, s2, s5) values('d1',31, 31, 'aligned_test31')", + "insert into table0(device,time, s2, s5) values('d1',32, 32, 'aligned_test32')", + "insert into table0(device,time, s2, s5) values('d1',33, 33, 'aligned_test33')", + "insert into table0(device,time, s2, s5) values('d1',34, 34, 'aligned_test34')", + "insert into table0(device,time, s2, s5) values('d1',35, 35, 'aligned_test35')", + "insert into table0(device,time, s2, s5) values('d1',36, 36, 'aligned_test36')", + "insert into table0(device,time, s2, s5) values('d1',37, 37, 'aligned_test37')", + "insert into table0(device,time, s2, s5) values('d1',38, 38, 'aligned_test38')", + "insert into table0(device,time, s2, s5) values('d1',39, 39, 'aligned_test39')", + "insert into table0(device,time, s2, s5) values('d1',40, 40, 'aligned_test40')", + "insert into table0(device,time, s2, s5) values('d2',31, 31, 'non_aligned_test31')", + "insert into table0(device,time, s2, s5) values('d2',32, 32, 'non_aligned_test32')", + "insert into table0(device,time, s2, s5) values('d2',33, 33, 'non_aligned_test33')", + "insert into table0(device,time, s2, s5) values('d2',34, 34, 'non_aligned_test34')", + "insert into table0(device,time, s2, s5) values('d2',35, 35, 'non_aligned_test35')", + "insert into table0(device,time, s2, s5) values('d2',36, 36, 'non_aligned_test36')", + "insert into table0(device,time, s2, s5) values('d2',37, 37, 'non_aligned_test37')", + "insert into table0(device,time, s2, s5) values('d2',38, 38, 'non_aligned_test38')", + "insert into table0(device,time, s2, s5) values('d2',39, 39, 'non_aligned_test39')", + "insert into table0(device,time, s2, s5) values('d2',40, 40, 'non_aligned_test40')", + }; + + public static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void insertDataWithSession() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (String sql : sqls) { + session.executeNonQueryStatement(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableResultSetEqualTest( + String sql, String[] expectedHeader, String[] expectedRetArray, String database) { + tableResultSetEqualTest( + sql, + expectedHeader, + expectedRetArray, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + database); + } + + public static void tableResultSetEqualTest( + String sql, + String[] expectedHeader, + String[] expectedRetArray, + String userName, + String password, + String database) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + + for (int i = 1; i <= expectedHeader.length; i++) { + if (i == 1) { + builder.append(resultSet.getLong(i)).append(","); + } else { + builder.append(resultSet.getString(i)).append(","); + } + } + assertEquals(expectedRetArray[cnt], builder.toString()); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBCastFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBCastFunctionTableIT.java new file mode 100644 index 0000000000000..ee3bf466688cb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBCastFunctionTableIT.java @@ -0,0 +1,1371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCastFunctionTableIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] SQLs = + new String[] { + // normal cases + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table normal(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 STRING FIELD)", + // data for int series + "INSERT INTO normal(time, device_id ,s1) values(0, 'd1', 0)", + "INSERT INTO normal(time, device_id ,s1) values(1, 'd1', 1)", + "INSERT INTO normal(time, device_id ,s1) values(2, 'd1', 2)", + "INSERT INTO normal(time, device_id ,s1) values(3, 'd1', 3)", + // data for long series + "INSERT INTO normal(time, device_id ,s2) values(0, 'd1', 0)", + "INSERT INTO normal(time, device_id ,s2) values(1, 'd1', 1)", + "INSERT INTO normal(time, device_id ,s2) values(2, 'd1', 2)", + "INSERT INTO normal(time, device_id ,s2) values(3, 'd1', 3)", + // data for float series + "INSERT INTO normal(time, device_id ,s3) values(0, 'd1', 0)", + "INSERT INTO normal(time, device_id ,s3) values(1, 'd1', 1)", + "INSERT INTO normal(time, device_id ,s3) values(2, 'd1', 2.7)", + "INSERT INTO normal(time, device_id ,s3) values(3, 'd1', 3.33)", + // data for double series + "INSERT INTO normal(time, device_id ,s4) values(0, 'd1', 0)", + "INSERT INTO normal(time, device_id ,s4) values(1, 'd1', 1.0)", + "INSERT INTO normal(time, device_id ,s4) values(2, 'd1', 2.7)", + "INSERT INTO normal(time, device_id ,s4) values(3, 'd1', 3.33)", + // data for boolean series + "INSERT INTO normal(time, device_id ,s5) values(0, 'd1', false)", + "INSERT INTO normal(time, device_id ,s5) values(1, 'd1', false)", + "INSERT INTO normal(time, device_id ,s5) values(2, 'd1', true)", + "INSERT INTO normal(time, device_id ,s5) values(3, 'd1', true)", + // data for text series + "INSERT INTO normal(time, device_id ,s6) values(0, 'd1', '10000')", + "INSERT INTO normal(time, device_id ,s6) values(1, 'd1', 3)", + "INSERT INTO normal(time, device_id ,s6) values(2, 'd1', 'TRue')", + "INSERT INTO normal(time, device_id ,s6) values(3, 'd1', 'faLse')", + "flush", + + // special cases + "create table special(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 STRING FIELD)", + "INSERT INTO special(Time, device_id ,s2) values(1, 'd1', 2147483648)", + "INSERT INTO special(Time, device_id ,s3) values(1, 'd1', 2147483648.0)", + "INSERT INTO special(Time, device_id ,s3) values(2, 'd1', 2e38)", + "INSERT INTO special(Time, device_id ,s4) values(1, 'd1', 4e50)", + "INSERT INTO special(Time, device_id ,s6) values(1, 'd1', 'test')", + "INSERT INTO special(Time, device_id ,s6) values(2, 'd1', '1.1')", + "INSERT INTO special(Time, device_id ,s6) values(3, 'd1', '4e60')", + "INSERT INTO special(Time, device_id ,s6) values(4, 'd1', '4e60000')", + "flush", + + // special cases for date and timestamp + "create table dateType(device_id STRING TAG, s1 DATE FIELD, s2 TIMESTAMP FIELD)", + "INSERT INTO dateType(Time,device_id, s1, s2) values(1,'d1', '9999-12-31', 253402271999999)", + "INSERT INTO dateType(Time,device_id, s1, s2) values(2,'d1', '1000-01-01', -30610224000000)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // region ================== New Transformer ================== + @Test + public void testNewTransformerWithIntSource() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s1"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s1 AS INT32) as cast_s1 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s1 AS INT64) as cast_s1 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s1 AS FLOAT) as cast_s1 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s1 AS DOUBLE) as cast_s1 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, CAST(s1 AS BOOLEAN) as cast_s1 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to string + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s1 AS STRING) as cast_s1 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithLongSource() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s2"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + + tableResultSetEqualTest( + "select Time, CAST(s2 AS INT32) as cast_s2 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s2 AS INT64) as cast_s2 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s2 AS FLOAT) as cast_s2 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s2 AS DOUBLE) as cast_s2 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, CAST(s2 AS BOOLEAN) as cast_s2 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s2 AS STRING) as cast_s2 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithFloatSource() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s3"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s3 AS INT32) as cast_s3 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s3 AS INT64) as cast_s3 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, CAST(s3 AS FLOAT) as cast_s3 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.700000047683716,", + "1970-01-01T00:00:00.003Z,3.3299999237060547,", + }; + tableResultSetEqualTest( + "select Time, CAST(s3 AS DOUBLE) as cast_s3 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, CAST(s3 AS BOOLEAN) as cast_s3 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, CAST(s3 AS STRING) as cast_s3 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithDoubleSource() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s4"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s4 AS INT32) as cast_s4 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s4 AS INT64) as cast_s4 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, CAST(s4 AS FLOAT) as cast_s4 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, CAST(s4 AS DOUBLE) as cast_s4 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, CAST(s4 AS BOOLEAN) as cast_s4 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to string + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, CAST(s4 AS STRING) as cast_s4 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithBooleanSource() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s5"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,0,", + "1970-01-01T00:00:00.002Z,1,", + "1970-01-01T00:00:00.003Z,1,", + }; + tableResultSetEqualTest( + "select Time, CAST(s5 AS INT32) as cast_s5 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,0,", + "1970-01-01T00:00:00.002Z,1,", + "1970-01-01T00:00:00.003Z,1,", + }; + tableResultSetEqualTest( + "select Time, CAST(s5 AS INT64) as cast_s5 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,1.0,", + "1970-01-01T00:00:00.003Z,1.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s5 AS FLOAT) as cast_s5 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,1.0,", + "1970-01-01T00:00:00.003Z,1.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s5 AS DOUBLE) as cast_s5 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, CAST(s5 AS BOOLEAN) as cast_s5 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, CAST(s5 AS STRING) as cast_s5 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithTextSource() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s6"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000,", "1970-01-01T00:00:00.001Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s6 AS INT32) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000,", "1970-01-01T00:00:00.001Z,3,", + }; + tableResultSetEqualTest( + "select Time, CAST(s6 AS INT64) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000.0,", "1970-01-01T00:00:00.001Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s6 AS FLOAT) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000.0,", "1970-01-01T00:00:00.001Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, CAST(s6 AS DOUBLE) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.002Z,true,", "1970-01-01T00:00:00.003Z,false,", + }; + tableResultSetEqualTest( + "select Time, CAST(s6 AS BOOLEAN) as cast_s6 from normal where device_id='d1' and Time >= 2", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000,", "1970-01-01T00:00:00.001Z,3,", + "1970-01-01T00:00:00.002Z,TRue,", "1970-01-01T00:00:00.003Z,faLse,", + }; + tableResultSetEqualTest( + "select Time, CAST(s6 AS STRING) as cast_s6 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + // endregion + + // region try_cast + + @Test + public void testNewTransformerWithIntSourceInTryCast() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s1"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s1 AS INT32) as cast_s1 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s1 AS INT64) as cast_s1 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s1 AS FLOAT) as cast_s1 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s1 AS DOUBLE) as cast_s1 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s1 AS BOOLEAN) as cast_s1 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to string + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s1 AS STRING) as cast_s1 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithLongSourceInTryCast() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s2"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + + tableResultSetEqualTest( + "select Time, TRY_CAST(s2 AS INT32) as cast_s2 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s2 AS INT64) as cast_s2 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s2 AS FLOAT) as cast_s2 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s2 AS DOUBLE) as cast_s2 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s2 AS BOOLEAN) as cast_s2 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,2,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s2 AS STRING) as cast_s2 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithFloatSourceInTryCast() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s3"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s3 AS INT32) as cast_s3 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s3 AS INT64) as cast_s3 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s3 AS FLOAT) as cast_s3 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.700000047683716,", + "1970-01-01T00:00:00.003Z,3.3299999237060547,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s3 AS DOUBLE) as cast_s3 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s3 AS BOOLEAN) as cast_s3 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s3 AS STRING) as cast_s3 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithDoubleSourceInTryCast() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s4"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s4 AS INT32) as cast_s4 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,1,", + "1970-01-01T00:00:00.002Z,3,", + "1970-01-01T00:00:00.003Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s4 AS INT64) as cast_s4 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s4 AS FLOAT) as cast_s4 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s4 AS DOUBLE) as cast_s4 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,true,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s4 AS BOOLEAN) as cast_s4 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to string + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,2.7,", + "1970-01-01T00:00:00.003Z,3.33,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s4 AS STRING) as cast_s4 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithBooleanSourceInTryCast() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s5"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,0,", + "1970-01-01T00:00:00.002Z,1,", + "1970-01-01T00:00:00.003Z,1,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s5 AS INT32) as cast_s5 from normal where device_id='d1'", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0,", + "1970-01-01T00:00:00.001Z,0,", + "1970-01-01T00:00:00.002Z,1,", + "1970-01-01T00:00:00.003Z,1,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s5 AS INT64) as cast_s5 from normal where device_id='d1'", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,1.0,", + "1970-01-01T00:00:00.003Z,1.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s5 AS FLOAT) as cast_s5 from normal where device_id='d1'", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,1.0,", + "1970-01-01T00:00:00.003Z,1.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s5 AS DOUBLE) as cast_s5 from normal where device_id='d1'", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s5 AS BOOLEAN) as cast_s5 from normal where device_id='d1'", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,false,", + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,true,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s5 AS STRING) as cast_s5 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerWithTextSourceInTryCast() { + // cast to int + String[] expectedHeader = new String[] {TIMESTAMP_STR, "cast_s6"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000,", "1970-01-01T00:00:00.001Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s6 AS INT32) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + intRetArray, + DATABASE_NAME); + + // cast to long + String[] longRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000,", "1970-01-01T00:00:00.001Z,3,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s6 AS INT64) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + longRetArray, + DATABASE_NAME); + + // cast to float + String[] floatRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000.0,", "1970-01-01T00:00:00.001Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s6 AS FLOAT) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + floatRetArray, + DATABASE_NAME); + + // cast to double + String[] doubleRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000.0,", "1970-01-01T00:00:00.001Z,3.0,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s6 AS DOUBLE) as cast_s6 from normal where device_id='d1' and Time < 2", + expectedHeader, + doubleRetArray, + DATABASE_NAME); + + // cast to boolean + String[] booleanRetArray = + new String[] { + "1970-01-01T00:00:00.002Z,true,", "1970-01-01T00:00:00.003Z,false,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s6 AS BOOLEAN) as cast_s6 from normal where device_id='d1' and Time >= 2", + expectedHeader, + booleanRetArray, + DATABASE_NAME); + + // cast to text + String[] stringRetArray = + new String[] { + "1970-01-01T00:00:00.000Z,10000,", "1970-01-01T00:00:00.001Z,3,", + "1970-01-01T00:00:00.002Z,TRue,", "1970-01-01T00:00:00.003Z,faLse,", + }; + tableResultSetEqualTest( + "select Time, TRY_CAST(s6 AS STRING) as cast_s6 from normal where device_id='d1'", + expectedHeader, + stringRetArray, + DATABASE_NAME); + } + + // endregion + + // region special cases + + @Test + public void testCastWithLongSource() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + try { + statement.execute("USE " + DATABASE_NAME); + statement.execute("select CAST(s2 AS INT32) from special where device_id='d1'"); + fail(); + } catch (Exception ignored) { + } + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void testTryCastWithLongSource() { + String sql = "select TRY_CAST(s2 AS INT32) from special where device_id='d1'"; + tableResultSetEqualTest( + sql, + new String[] {"_col0"}, + new String[] {"null,", "null,", "null,", "null,"}, + DATABASE_NAME); + } + + @Test + public void testCastWithFloatSource() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + try { + statement.execute("USE " + DATABASE_NAME); + statement.execute("select CAST(s3 AS INT32) from special where device_id='d1'"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("USE " + DATABASE_NAME); + statement.execute("select CAST(s3 AS INT64) from special where device_id='d1'"); + fail(); + } catch (Exception ignored) { + } + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void testTryCastWithFloatSource() { + String sql = "select TRY_CAST(s3 AS INT32) from special where device_id='d1'"; + tableResultSetEqualTest( + sql, + new String[] {"_col0"}, + new String[] {"2147483647,", "null,", "null,", "null,"}, + DATABASE_NAME); + } + + @Test + public void testCastWithDoubleSource() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + try { + statement.execute("USE " + DATABASE_NAME); + statement.execute("select CAST(s4 AS INT32) from special where device_id='d1'"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("USE " + DATABASE_NAME); + statement.execute("select CAST(s4 AS INT64) from special where device_id='d1'"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("USE " + DATABASE_NAME); + statement.execute("select CAST(s4 AS Float) from special where device_id='d1'"); + fail(); + } catch (Exception ignored) { + } + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void testTryCastWithDoubleSource() { + String sql = "select TRY_CAST(s4 AS Float) from special where device_id='d1'"; + tableResultSetEqualTest( + sql, + new String[] {"_col0"}, + new String[] {"null,", "null,", "null,", "null,"}, + DATABASE_NAME); + } + + @Test + public void testCastWithTextSource() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try { + statement.execute( + "select CAST(s6 AS INT32) from special where device_id='d1' and time = 1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute( + "select CAST(s6 AS INT32) from special where device_id='d1' and time = 2"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute( + "select CAST(s6 AS INT64) from special where device_id='d1' and time = 1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute( + "select CAST(s6 AS INT64) from special where device_id='d1' and time = 2"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select CAST(s6 AS FLOAT) from special where device_id='d1' and time=3"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select CAST(s6 AS FLOAT) from special where device_id='d1' and time=1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute( + "select CAST(s6 AS DOUBLE) from special where device_id='d1' and time = 1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute( + "select CAST(s6 AS DOUBLE) from special where device_id='d1' and time = 4"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute( + "select CAST(s6 AS BOOLEAN) from special where device_id='d1' and time = 1"); + fail(); + } catch (Exception ignored) { + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void testTryCastWithTextSource() { + String sql = "select TRY_CAST(s6 AS INT32) from special where device_id='d1' and time = 1"; + tableResultSetEqualTest(sql, new String[] {"_col0"}, new String[] {"null,"}, DATABASE_NAME); + } + + @Test + public void testDateOutOfRange() { + tableAssertTestFail( + String.format( + "select CAST((s1 + %d) AS TIMESTAMP) from dateType where time = 1", + DateTimeUtils.correctPrecision(86400000)), + "752: Year must be between 1000 and 9999.", + DATABASE_NAME); + + tableAssertTestFail( + String.format( + "select CAST((s2 + %d) AS DATE) from dateType where time = 1", + DateTimeUtils.correctPrecision(86400000)), + "752: Year must be between 1000 and 9999.", + DATABASE_NAME); + + tableAssertTestFail( + String.format( + "select CAST((s1 - %d) AS TIMESTAMP) from dateType where time = 2", + DateTimeUtils.correctPrecision(86400000)), + "752: Year must be between 1000 and 9999.", + DATABASE_NAME); + + tableAssertTestFail( + String.format( + "select CAST((s2 - %d) AS DATE) from dateType where time = 2", + DateTimeUtils.correctPrecision(86400000)), + "752: Year must be between 1000 and 9999.", + DATABASE_NAME); + } + + // endregion +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBCastFunctionTableSpecialIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBCastFunctionTableSpecialIT.java new file mode 100644 index 0000000000000..5620f691924c2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBCastFunctionTableSpecialIT.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCastFunctionTableSpecialIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table special(device_id STRING TAG, s1 DATE FIELD)", + "INSERT INTO special(time, device_id, s1) values(1, 'd1', '2262-04-13')", + "INSERT INTO special(time, device_id, s1) values(2, 'd1', '1677-09-21')", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("ns"); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testTimestampOverflow() { + tableAssertTestFail( + "select cast(s1 as TIMESTAMP) from special where time = 1", + "750: Timestamp overflow", + DATABASE_NAME); + + tableAssertTestFail( + "select cast(s1 as TIMESTAMP) from special where time = 2", + "750: Timestamp overflow", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTable2IT.java new file mode 100644 index 0000000000000..9976d83055e0f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTable2IT.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDiffFunctionTable2IT extends IoTDBDiffFunctionTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDataRegionGroupExtensionPolicy("CUSTOM"); + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultDataRegionGroupNumPerDatabase(2); + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTable3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTable3IT.java new file mode 100644 index 0000000000000..1f8278fef313a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTable3IT.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDiffFunctionTable3IT extends IoTDBDiffFunctionTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDataRegionGroupExtensionPolicy("CUSTOM"); + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultDataRegionGroupNumPerDatabase(3); + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java new file mode 100644 index 0000000000000..6bf1ef8db3e86 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDiffFunctionTableIT { + + private static final String DATABASE_NAME = "db"; + + // 2 devices 4 regions + protected static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create table table1(device_id STRING TAG, s1 INT32 FIELD, s2 FLOAT FIELD)", + "INSERT INTO table1(time,device_id,s1,s2) values(1, 'd1', 1, 1)", + "INSERT INTO table1(time,device_id,s1) values(2, 'd1', 2)", + "INSERT INTO table1(time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO table1(time,device_id,s1) values(4, 'd1', 4)", + "INSERT INTO table1(time,device_id,s1,s2) values(5, 'd1', 5, 5)", + "INSERT INTO table1(time,device_id,s2) values(6, 'd1', 6)", + "INSERT INTO table1(time,device_id,s1,s2) values(5000000000, 'd1', null, 7)", + "INSERT INTO table1(time,device_id,s1,s2) values(5000000001, 'd1', 8, null)", + "INSERT INTO table1(time,device_id,s1,s2) values(1, 'd2', 1, 1)", + "INSERT INTO table1(time,device_id,s1,s2) values(2, 'd2', 2, 2)", + "INSERT INTO table1(time,device_id,s1,s2) values(5000000000, 'd2', null, 3)", + "INSERT INTO table1(time,device_id,s1,s2) values(5000000001, 'd2', 4, null)", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testNewTransformerIgnoreNull() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,null,null,", + "1970-01-01T00:00:00.002Z,1.0,null,", + "1970-01-01T00:00:00.003Z,null,2.0,", + "1970-01-01T00:00:00.004Z,2.0,null,", + "1970-01-01T00:00:00.005Z,1.0,2.0,", + "1970-01-01T00:00:00.006Z,null,1.0,", + "1970-02-27T20:53:20.000Z,null,1.0,", + "1970-02-27T20:53:20.001Z,3.0,null," + }; + tableResultSetEqualTest( + "select time,diff(s1), diff(s2) from table1 where device_id='d1'", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNewTransformerRespectNull() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,null,null,", + "1970-01-01T00:00:00.002Z,1.0,null,", + "1970-01-01T00:00:00.003Z,null,null,", + "1970-01-01T00:00:00.004Z,null,null,", + "1970-01-01T00:00:00.005Z,1.0,null,", + "1970-01-01T00:00:00.006Z,null,1.0,", + "1970-02-27T20:53:20.000Z,null,1.0,", + "1970-02-27T20:53:20.001Z,null,null," + }; + tableResultSetEqualTest( + "select time, Diff(s1, false), diff(s2, false) from table1 where device_id='d1'", + expectedHeader, + retArray, + DATABASE_NAME); + + // align by device + order by time + } + + @Test + public void testCaseInSensitive() { + String[] expectedHeader = new String[] {"time", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.002Z,2,null,", + "1970-01-01T00:00:00.004Z,4,null,", + "1970-01-01T00:00:00.005Z,5,5.0," + }; + tableResultSetEqualTest( + "select time, s1, s2 from table1 where device_id='d1' and diff(s1) between 1 and 2", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java new file mode 100644 index 0000000000000..f8bde5af7baf9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java @@ -0,0 +1,217 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.time.LocalDate; +import java.time.ZoneId; + +import static java.lang.String.format; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFormatFunctionTableIT { + + private static final String DATABASE_NAME = "db"; + + private static final ZoneId zoneId = ZoneId.of("Z"); + + private static final String[] SQLs = + new String[] { + // normal cases + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE number_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD)", + "CREATE TABLE string_table(device_id STRING TAG, s1 STRING FIELD, s2 TEXT FIELD)", + "CREATE TABLE boolean_table(device_id STRING TAG, s1 BOOLEAN FIELD)", + "CREATE TABLE timestamp_table(device_id STRING TAG, s1 TIMESTAMP FIELD)", + "CREATE TABLE date_table(device_id STRING TAG, s1 DATE FIELD)", + "CREATE TABLE null_table(device_id STRING TAG, s1 INT32 FIELD)", + // data for number series + "INSERT INTO number_table(time, device_id, s1, s2, s3, s4) VALUES (10, 'd1', 1000000, 1000000, 3.1415926, 3.1415926)", + "INSERT INTO number_table(time, device_id, s1, s2, s3, s4) VALUES (20, 'd1', 1, 1, 1234567.25, 1234567.25)", + + // data for string series + "INSERT INTO string_table(time, device_id, s1, s2) VALUES (10, 'd1', 'world', 'world')", + "INSERT INTO string_table(time, device_id, s1, s2) VALUES (20, 'd1', '123', '123')", + + // data for boolean series + "INSERT INTO boolean_table(time, device_id, s1) VALUES (10, 'd1', 'true')", + "INSERT INTO boolean_table(time, device_id, s1) VALUES (20, 'd1', 'false')", + + // data for timestamp series + "INSERT INTO timestamp_table(time, device_id, s1) VALUES (10, 'd1', 10)", + "INSERT INTO timestamp_table(time, device_id, s1) VALUES (20, 'd1', 20)", + + // data for date series + "INSERT INTO date_table(time, device_id, s1) VALUES (10, 'd1', '2024-01-01')", + "INSERT INTO date_table(time, device_id, s1) VALUES (20, 'd1', '2006-07-04')", + "INSERT INTO date_table(time, device_id, s1) VALUES (30, 'd2', '1970-01-01')", + "INSERT INTO date_table(time, device_id, s1) VALUES (40, 'd2', '9999-12-31')", + "INSERT INTO date_table(time, device_id, s1) VALUES (50, 'd2', '1900-01-01')", + + // data for special series + "INSERT INTO null_table(time, device_id, s1) VALUES (10, 'd1', null)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("ns"); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testStringFormat() { + tableResultSetEqualTest( + "SELECT FORMAT('hello %s', s1),FORMAT('hello %s', s2) FROM string_table WHERE time = 10", + new String[] {"_col0", "_col1"}, new String[] {"hello world,hello world,"}, DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%s%%', s1),FORMAT('%s%%', s1) FROM string_table WHERE time = 20", + new String[] {"_col0", "_col1"}, new String[] {"123%,123%,"}, DATABASE_NAME); + } + + @Test + public void testBooleanFormat() { + tableResultSetEqualTest( + "SELECT FORMAT('%1$b %1$B', s1) FROM boolean_table", + new String[] {"_col0"}, new String[] {"true TRUE,", "false FALSE,"}, DATABASE_NAME); + } + + @Test + public void testNumberFormat() { + tableResultSetEqualTest( + "SELECT FORMAT('%,d %,d', s1, s2) FROM number_table WHERE time = 10", + new String[] {"_col0"}, new String[] {"1,000,000 1,000,000,"}, DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%.2f %.2f', s3, s4) FROM number_table WHERE time = 10", + new String[] {"_col0"}, new String[] {"3.14 3.14,"}, DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%03d %03d', s1, s2) FROM number_table WHERE time = 20", + new String[] {"_col0"}, new String[] {"001 001,"}, DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%,.2f %,.2f', s3, s4) FROM number_table WHERE time = 20", + new String[] {"_col0"}, new String[] {"1,234,567.25 1,234,567.25,"}, DATABASE_NAME); + } + + @Test + public void testTimestampFormat() { + tableResultSetEqualTest( + "SELECT FORMAT('%1$tF %1$tT', s1) FROM timestamp_table", + new String[] {"_col0"}, + new String[] {"1970-01-01 00:00:00,", "1970-01-01 00:00:00,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%1$tF %1$tT.%1tL', s1) FROM timestamp_table", + new String[] {"_col0"}, + new String[] {"1970-01-01 00:00:00.000,", "1970-01-01 00:00:00.000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%tc', s1) FROM timestamp_table", + new String[] {"_col0"}, + new String[] { + format("%tc,", DateTimeUtils.convertToZonedDateTime(10, zoneId)), + format("%tc,", DateTimeUtils.convertToZonedDateTime(20, zoneId)), + }, + DATABASE_NAME); + } + + @Test + public void testDateFormat() { + tableResultSetEqualTest( + "SELECT FORMAT('%1$tA, %1$tB %1$te, %1$tY', s1) FROM date_table where device_id = 'd1'", + new String[] {"_col0"}, + new String[] { + format("%1$tA, %1$tB %1$te, %1$tY,", LocalDate.of(2024, 1, 1)), + format("%1$tA, %1$tB %1$te, %1$tY,", LocalDate.of(2006, 7, 4)) + }, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%1$s %1$tF %1$tY-%1$tm-%1$td', s1) FROM date_table where device_id = 'd1'", + new String[] {"_col0"}, + new String[] {"2024-01-01 2024-01-01 2024-01-01,", "2006-07-04 2006-07-04 2006-07-04,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT FORMAT('%1$tY', s1) FROM date_table where device_id = 'd2'", + new String[] {"_col0"}, new String[] {"1970,", "9999,", "1900,"}, DATABASE_NAME); + } + + @Test + public void testNullFormat() { + tableResultSetEqualTest( + "SELECT FORMAT('Value:%s', s1) FROM null_table", + new String[] {"_col0"}, new String[] {"Value:null,"}, DATABASE_NAME); + } + + @Test + public void testAnomalies() { + tableAssertTestFail( + "SELECT FORMAT('%1$tA, %1$tB %1$te, %1$tY', s1) FROM number_table", + "701: Invalid format string:", DATABASE_NAME); + + tableAssertTestFail( + "SELECT FORMAT('%f', s1) FROM string_table", "701: Invalid format string:", DATABASE_NAME); + + tableAssertTestFail( + "SELECT FORMAT('%s %d', s1) FROM string_table", + "701: Invalid format string:", DATABASE_NAME); + + tableAssertTestFail( + "SELECT FORMAT('%s') FROM string_table", + "701: Scalar function format must have at least two arguments, and first argument pattern must be TEXT or STRING type.", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBReplaceFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBReplaceFunctionTableIT.java new file mode 100644 index 0000000000000..210de9565d18b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBReplaceFunctionTableIT.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBReplaceFunctionTableIT { + + private static final String DATABASE_NAME = "db"; + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create table table1(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd')", + "INSERT INTO table1(time,device_id,s1) values(2, 'd1', 'test\\\\')", + "INSERT INTO table1(time,device_id,s1) values(3, 'd1', 'abcd\\\\')", + "INSERT INTO table1(time,device_id,s9) values(2, 'd1', 'test\\\\')", + "INSERT INTO table1(time,device_id,s9) values(3, 'd1', 'abcd\\\\')", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testNewTransformer() { + String[] expectedHeader = new String[] {TIMESTAMP_STR, "_col1", "_col2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,ABcd,abcd,", + "1970-01-01T00:00:00.002Z,test\\\\,testaa,", + "1970-01-01T00:00:00.003Z,ABcd\\\\,abcdaa," + }; + tableResultSetEqualTest( + "select Time, REPLACE(s1, 'ab', 'AB'), REPLACE(s1, '\\', 'a') from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.001Z,ABcd,abcd,", + "1970-01-01T00:00:00.002Z,test\\\\,testaa,", + "1970-01-01T00:00:00.003Z,ABcd\\\\,abcdaa," + }; + tableResultSetEqualTest( + "select Time, REPLACE(s9, 'ab', 'AB'), REPLACE(s9, '\\', 'a') from table1", + expectedHeader, + retArray2, + DATABASE_NAME); + } + + @Test + public void testWithoutTo() { + String[] expectedHeader = new String[] {TIMESTAMP_STR, "_col1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,acd,", + "1970-01-01T00:00:00.002Z,test\\\\,", + "1970-01-01T00:00:00.003Z,acd\\\\," + }; + tableResultSetEqualTest( + "select Time, REPLACE(s1, 'b') from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testWrongInputType() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try { + statement.execute("select REPLACE(s2, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select REPLACE(s3, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select REPLACE(s4, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select REPLACE(s5, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select REPLACE(s6, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select REPLACE(s7, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + try { + statement.execute("select REPLACE(s8, 'a', 'b') from table1"); + fail(); + } catch (Exception ignored) { + } + + // try { + // statement.execute("select REPLACE(s10, 'a', 'b') from table1"); + // fail(); + // } catch (Exception ignored) { + // } + + } catch (SQLException e) { + e.printStackTrace(); + fail(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java new file mode 100644 index 0000000000000..8c0c74bf36a9f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRoundFunctionTableIT { + + private static final String DATABASE_NAME = "db"; + + protected static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create table table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 DOUBLE FIELD, s4 FLOAT FIELD, s5 BOOLEAN FIELD, s6 STRING FIELD)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(1, 'd1', 2, 3, 0.11234, 101.143445345,true,null)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(2, 'd1', 2, 4, 10.11234, 20.1443465345,true,'sss')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(3, 'd1', 2, 555, 120.161234, 20.61437245345,true,'sss')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(4, 'd1', 2, 12341234, 101.131234, null,true,'sss')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(5, 'd1', 2, 55678, 90.116234, 20.8143454345,true,'sss')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(6, 'd1', 2, 12355, null, 60.71443345345,true,'sss')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6) values(1678695379764, 'd1', 2, 12345, 120.511234, 10.143425345,null,'sss')", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testRound() { + String[] expectedHeader = new String[] {"time", "_col1"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,2.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,2.0,", + "1970-01-01T00:00:00.004Z,2.0,", + "1970-01-01T00:00:00.005Z,2.0,", + "1970-01-01T00:00:00.006Z,2.0,", + "2023-03-13T08:16:19.764Z,2.0," + }; + tableResultSetEqualTest( + "select time, round(s1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,3.0,", + "1970-01-01T00:00:00.002Z,4.0,", + "1970-01-01T00:00:00.003Z,555.0,", + "1970-01-01T00:00:00.004Z,1.2341234E7,", + "1970-01-01T00:00:00.005Z,55678.0,", + "1970-01-01T00:00:00.006Z,12355.0,", + "2023-03-13T08:16:19.764Z,12345.0," + }; + tableResultSetEqualTest( + "select time, round(s2) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,10.0,", + "1970-01-01T00:00:00.003Z,120.0,", + "1970-01-01T00:00:00.004Z,101.0,", + "1970-01-01T00:00:00.005Z,90.0,", + "1970-01-01T00:00:00.006Z,null,", + "2023-03-13T08:16:19.764Z,121.0," + }; + tableResultSetEqualTest( + "select time, round(s3) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,101.0,", + "1970-01-01T00:00:00.002Z,20.0,", + "1970-01-01T00:00:00.003Z,21.0,", + "1970-01-01T00:00:00.004Z,null,", + "1970-01-01T00:00:00.005Z,21.0,", + "1970-01-01T00:00:00.006Z,61.0,", + "2023-03-13T08:16:19.764Z,10.0," + }; + tableResultSetEqualTest( + "select time, round(s4) from table1", expectedHeader, intRetArray, DATABASE_NAME); + } + + @Test + public void testRoundWithPlaces() { + String[] expectedHeader = new String[] {"time", "_col1"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,2.0,", + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,2.0,", + "1970-01-01T00:00:00.004Z,2.0,", + "1970-01-01T00:00:00.005Z,2.0,", + "1970-01-01T00:00:00.006Z,2.0,", + "2023-03-13T08:16:19.764Z,2.0," + }; + tableResultSetEqualTest( + "select time, round(s1, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,3.0,", + "1970-01-01T00:00:00.002Z,4.0,", + "1970-01-01T00:00:00.003Z,555.0,", + "1970-01-01T00:00:00.004Z,1.2341234E7,", + "1970-01-01T00:00:00.005Z,55678.0,", + "1970-01-01T00:00:00.006Z,12355.0,", + "2023-03-13T08:16:19.764Z,12345.0," + }; + tableResultSetEqualTest( + "select time, round(s2, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.1,", + "1970-01-01T00:00:00.002Z,10.1,", + "1970-01-01T00:00:00.003Z,120.2,", + "1970-01-01T00:00:00.004Z,101.1,", + "1970-01-01T00:00:00.005Z,90.1,", + "1970-01-01T00:00:00.006Z,null,", + "2023-03-13T08:16:19.764Z,120.5," + }; + tableResultSetEqualTest( + "select time, round(s3, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.11,", + "1970-01-01T00:00:00.002Z,10.11,", + "1970-01-01T00:00:00.003Z,120.16,", + "1970-01-01T00:00:00.004Z,101.13,", + "1970-01-01T00:00:00.005Z,90.12,", + "1970-01-01T00:00:00.006Z,null,", + "2023-03-13T08:16:19.764Z,120.51," + }; + tableResultSetEqualTest( + "select time, round(s3, 2) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.112,", + "1970-01-01T00:00:00.002Z,10.112,", + "1970-01-01T00:00:00.003Z,120.161,", + "1970-01-01T00:00:00.004Z,101.131,", + "1970-01-01T00:00:00.005Z,90.116,", + "1970-01-01T00:00:00.006Z,null,", + "2023-03-13T08:16:19.764Z,120.511," + }; + tableResultSetEqualTest( + "select time, round(s3, 3) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.1123,", + "1970-01-01T00:00:00.002Z,10.1123,", + "1970-01-01T00:00:00.003Z,120.1612,", + "1970-01-01T00:00:00.004Z,101.1312,", + "1970-01-01T00:00:00.005Z,90.1162,", + "1970-01-01T00:00:00.006Z,null,", + "2023-03-13T08:16:19.764Z,120.5112," + }; + tableResultSetEqualTest( + "select time, round(s3, 4) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,101.14345,", + "1970-01-01T00:00:00.002Z,20.14435,", + "1970-01-01T00:00:00.003Z,20.61437,", + "1970-01-01T00:00:00.004Z,null,", + "1970-01-01T00:00:00.005Z,20.81435,", + "1970-01-01T00:00:00.006Z,60.71443,", + "2023-03-13T08:16:19.764Z,10.14342," + }; + tableResultSetEqualTest( + "select time, round(s4, 5) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,101.1,", + "1970-01-01T00:00:00.002Z,20.1,", + "1970-01-01T00:00:00.003Z,20.6,", + "1970-01-01T00:00:00.004Z,null,", + "1970-01-01T00:00:00.005Z,20.8,", + "1970-01-01T00:00:00.006Z,60.7,", + "2023-03-13T08:16:19.764Z,10.1," + }; + tableResultSetEqualTest( + "select time, round(s4, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + } + + @Test + public void testRoundWithNegativePlaces() { + + String[] expectedHeader = new String[] {"time", "_col1"}; + String[] intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,0.0,", + "1970-01-01T00:00:00.003Z,0.0,", + "1970-01-01T00:00:00.004Z,0.0,", + "1970-01-01T00:00:00.005Z,0.0,", + "1970-01-01T00:00:00.006Z,0.0,", + "2023-03-13T08:16:19.764Z,0.0," + }; + tableResultSetEqualTest( + "select time, round(s1, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,0.0,", + "1970-01-01T00:00:00.003Z,560.0,", + "1970-01-01T00:00:00.004Z,1.234123E7,", + "1970-01-01T00:00:00.005Z,55680.0,", + "1970-01-01T00:00:00.006Z,12360.0,", + "2023-03-13T08:16:19.764Z,12340.0," + }; + tableResultSetEqualTest( + "select time, round(s2, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,0.0,", + "1970-01-01T00:00:00.002Z,10.0,", + "1970-01-01T00:00:00.003Z,120.0,", + "1970-01-01T00:00:00.004Z,100.0,", + "1970-01-01T00:00:00.005Z,90.0,", + "1970-01-01T00:00:00.006Z,null,", + "2023-03-13T08:16:19.764Z,120.0," + }; + tableResultSetEqualTest( + "select time, round(s3,-1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + + intRetArray = + new String[] { + "1970-01-01T00:00:00.001Z,100.0,", + "1970-01-01T00:00:00.002Z,20.0,", + "1970-01-01T00:00:00.003Z,20.0,", + "1970-01-01T00:00:00.004Z,null,", + "1970-01-01T00:00:00.005Z,20.0,", + "1970-01-01T00:00:00.006Z,60.0,", + "2023-03-13T08:16:19.764Z,10.0," + }; + tableResultSetEqualTest( + "select time, round(s4, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + } + + @Test + public void testRoundBooleanAndText() { + tableAssertTestFail( + "select round(s5) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function round only supports two numeric data types [INT32, INT64, FLOAT, DOUBLE]", + DATABASE_NAME); + + tableAssertTestFail( + "select round(s6) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function round only supports two numeric data types [INT32, INT64, FLOAT, DOUBLE]", + DATABASE_NAME); + + tableAssertTestFail( + "select round(time) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function round only supports two numeric data types [INT32, INT64, FLOAT, DOUBLE]", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java new file mode 100644 index 0000000000000..f6ca8918b1aeb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java @@ -0,0 +1,2872 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.*; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBScalarFunctionTableIT { + private static final String DATABASE_NAME = "db"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "use " + DATABASE_NAME, + // absSQL + "create table absTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO absTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO absTable(Time,device_id,s2) values(2, 'd1', -1)", + "INSERT INTO absTable(Time,device_id,s2) values(3, 'd1', -2)", + "INSERT INTO absTable(Time,device_id,s3) values(2, 'd1', -1)", + "INSERT INTO absTable(Time,device_id,s3) values(3, 'd1', -2)", + "INSERT INTO absTable(Time,device_id,s4) values(2, 'd1', -1.5)", + "INSERT INTO absTable(Time,device_id,s4) values(3, 'd1', -2.5)", + "INSERT INTO absTable(Time,device_id,s5) values(2, 'd1', -1.5)", + "INSERT INTO absTable(Time,device_id,s5) values(3, 'd1', -2.5)", + // acosSQL + "create table acosTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO acosTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1, 'abcd', X'abcd')", + "INSERT INTO acosTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO acosTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO acosTable(Time,device_id,s4) values(2, 'd1', 0.5)", + "INSERT INTO acosTable(Time,device_id,s5) values(2, 'd1', 0.5)", + // asinSQL + "create table asinTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO asinTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1, 'abcd', X'abcd')", + "INSERT INTO asinTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO asinTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO asinTable(Time,device_id,s4) values(2, 'd1', 0.5)", + "INSERT INTO asinTable(Time,device_id,s5) values(2, 'd1', 0.5)", + // atanSQL + "create table atanTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO atanTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO atanTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO atanTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO atanTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO atanTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO atanTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO atanTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO atanTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO atanTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // ceilSQL + "create table ceilTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO ceilTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO ceilTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO ceilTable(Time,device_id,s2) values(3, 'd1', -2)", + "INSERT INTO ceilTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO ceilTable(Time,device_id,s3) values(3, 'd1', -2)", + "INSERT INTO ceilTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO ceilTable(Time,device_id,s4) values(3, 'd1', -2.5)", + "INSERT INTO ceilTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO ceilTable(Time,device_id,s5) values(3, 'd1', -2.5)", + // concatSQL + "create table concatTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO concatTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO concatTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO concatTable(Time,device_id,s1) values(3, 'd1', 'efgh')", + "INSERT INTO concatTable(Time,device_id,s1) values(4, 'd1', null)", + "INSERT INTO concatTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO concatTable(Time,device_id,s9) values(3, 'd1', 'efgh')", + "INSERT INTO concatTable(Time,device_id,s9) values(4, 'd1', 'haha')", + // cosSQL + "create table cosTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO cosTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO cosTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO cosTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO cosTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO cosTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO cosTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO cosTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO cosTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO cosTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // coshSQL + "create table coshTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO coshTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO coshTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO coshTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO coshTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO coshTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO coshTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO coshTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO coshTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO coshTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // degreesSQL + "create table degreesTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO degreesTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO degreesTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO degreesTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO degreesTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO degreesTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO degreesTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO degreesTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO degreesTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO degreesTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // endsWithSQL + "create table endsWithTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO endsWithTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO endsWithTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO endsWithTable(Time,device_id,s1) values(3, 'd1', 'efgh')", + "INSERT INTO endsWithTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO endsWithTable(Time,device_id,s9) values(3, 'd1', 'efgh')", + // expSQL + "create table expTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO expTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO expTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO expTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO expTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO expTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO expTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO expTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO expTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO expTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // floorSQL + "create table floorTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO floorTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO floorTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO floorTable(Time,device_id,s2) values(3, 'd1', -2)", + "INSERT INTO floorTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO floorTable(Time,device_id,s3) values(3, 'd1', -2)", + "INSERT INTO floorTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO floorTable(Time,device_id,s4) values(3, 'd1', -2.5)", + "INSERT INTO floorTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO floorTable(Time,device_id,s5) values(3, 'd1', -2.5)", + // lengthSQL + "create table lengthTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO lengthTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO lengthTable(Time,device_id,s1) values(2, 'd1', 'test')", + "INSERT INTO lengthTable(Time,device_id,s1) values(3, 'd1', 'abcdefg')", + "INSERT INTO lengthTable(Time,device_id,s9) values(2, 'd1', 'test')", + "INSERT INTO lengthTable(Time,device_id,s9) values(3, 'd1', 'abcdefg')", + // lnSQL + "create table lnTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO lnTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO lnTable(Time,device_id,s2) values(2, 'd1', 0)", + "INSERT INTO lnTable(Time,device_id,s2) values(3, 'd1', -3)", + "INSERT INTO lnTable(Time,device_id,s3) values(2, 'd1', 0)", + "INSERT INTO lnTable(Time,device_id,s3) values(3, 'd1', -3)", + "INSERT INTO lnTable(Time,device_id,s4) values(2, 'd1', 0.0)", + "INSERT INTO lnTable(Time,device_id,s4) values(3, 'd1', -3.5)", + "INSERT INTO lnTable(Time,device_id,s5) values(2, 'd1', 0.0)", + "INSERT INTO lnTable(Time,device_id,s5) values(3, 'd1', -3.5)", + // log10SQL + "create table log10Table(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO log10Table(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO log10Table(Time,device_id,s2) values(2, 'd1', 0)", + "INSERT INTO log10Table(Time,device_id,s2) values(3, 'd1', -3)", + "INSERT INTO log10Table(Time,device_id,s3) values(2, 'd1', 0)", + "INSERT INTO log10Table(Time,device_id,s3) values(3, 'd1', -3)", + "INSERT INTO log10Table(Time,device_id,s4) values(2, 'd1', 0.0)", + "INSERT INTO log10Table(Time,device_id,s4) values(3, 'd1', -3.5)", + "INSERT INTO log10Table(Time,device_id,s5) values(2, 'd1', 0.0)", + "INSERT INTO log10Table(Time,device_id,s5) values(3, 'd1', -3.5)", + // lowerSQL + "create table lowerTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO lowerTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'ABCD', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ABCD', X'abcd')", + "INSERT INTO lowerTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO lowerTable(Time,device_id,s1) values(3, 'd1', 'Abcdefg')", + "INSERT INTO lowerTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO lowerTable(Time,device_id,s9) values(3, 'd1', 'Abcdefg')", + // radiansSQL + "create table radiansTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO radiansTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO radiansTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO radiansTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO radiansTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO radiansTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO radiansTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO radiansTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO radiansTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO radiansTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // regexpLikeSQL + "create table regexpLikeTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO regexpLikeTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO regexpLikeTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO regexpLikeTable(Time,device_id,s1) values(3, 'd1', 'efgh')", + "INSERT INTO regexpLikeTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO regexpLikeTable(Time,device_id,s9) values(3, 'd1', '[e-g]+')", + // signSQL + "create table signTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO signTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO signTable(Time,device_id,s2) values(2, 'd1', 0)", + "INSERT INTO signTable(Time,device_id,s2) values(3, 'd1', -1)", + "INSERT INTO signTable(Time,device_id,s3) values(2, 'd1', 0)", + "INSERT INTO signTable(Time,device_id,s3) values(3, 'd1', -1)", + "INSERT INTO signTable(Time,device_id,s4) values(2, 'd1', 0.0)", + "INSERT INTO signTable(Time,device_id,s4) values(3, 'd1', -1.0)", + "INSERT INTO signTable(Time,device_id,s5) values(2, 'd1', 0.0)", + "INSERT INTO signTable(Time,device_id,s5) values(3, 'd1', -1.0)", + // sinSQL + "create table sinTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO sinTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO sinTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO sinTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO sinTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO sinTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO sinTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO sinTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO sinTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO sinTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // sinhSQL + "create table sinhTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO sinhTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO sinhTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO sinhTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO sinhTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO sinhTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO sinhTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO sinhTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO sinhTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO sinhTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // sqrtSQL + "create table sqrtTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO sqrtTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO sqrtTable(Time,device_id,s2) values(2, 'd1', 0)", + "INSERT INTO sqrtTable(Time,device_id,s2) values(3, 'd1', -1)", + "INSERT INTO sqrtTable(Time,device_id,s3) values(2, 'd1', 0)", + "INSERT INTO sqrtTable(Time,device_id,s3) values(3, 'd1', -1)", + "INSERT INTO sqrtTable(Time,device_id,s4) values(2, 'd1', 0.0)", + "INSERT INTO sqrtTable(Time,device_id,s4) values(3, 'd1', -1.5)", + "INSERT INTO sqrtTable(Time,device_id,s5) values(2, 'd1', 0.0)", + "INSERT INTO sqrtTable(Time,device_id,s5) values(3, 'd1', -1.5)", + // startsWithSQL + "create table startsWithTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO startsWithTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO startsWithTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO startsWithTable(Time,device_id,s1) values(3, 'd1', 'efgh')", + "INSERT INTO startsWithTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO startsWithTable(Time,device_id,s9) values(3, 'd1', 'efgh')", + // strcmpSQL + "create table strcmpTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO strcmpTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO strcmpTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO strcmpTable(Time,device_id,s1) values(3, 'd1', 'efgh')", + "INSERT INTO strcmpTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO strcmpTable(Time,device_id,s9) values(3, 'd1', 'efgh')", + // strposSQL + "create table strposTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO strposTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO strposTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO strposTable(Time,device_id,s1) values(3, 'd1', 'efgh')", + "INSERT INTO strposTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO strposTable(Time,device_id,s9) values(3, 'd1', 'efgh')", + // tanSQL + "create table tanTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO tanTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO tanTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO tanTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO tanTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO tanTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO tanTable(Time,device_id,s4) values(2, 'd1', 1.57079632675)", + "INSERT INTO tanTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO tanTable(Time,device_id,s5) values(2, 'd1', 1.57079632675)", + "INSERT INTO tanTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // tanhSQL + "create table tanhTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO tanhTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO tanhTable(Time,device_id,s2) values(2, 'd1', 2)", + "INSERT INTO tanhTable(Time,device_id,s2) values(3, 'd1', 3)", + "INSERT INTO tanhTable(Time,device_id,s3) values(2, 'd1', 2)", + "INSERT INTO tanhTable(Time,device_id,s3) values(3, 'd1', 3)", + "INSERT INTO tanhTable(Time,device_id,s4) values(2, 'd1', 2.5)", + "INSERT INTO tanhTable(Time,device_id,s4) values(3, 'd1', 3.5)", + "INSERT INTO tanhTable(Time,device_id,s5) values(2, 'd1', 2.5)", + "INSERT INTO tanhTable(Time,device_id,s5) values(3, 'd1', 3.5)", + // trimSQL + "create table trimTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO trimTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'ab', X'abcd')", + "INSERT INTO trimTable(Time,device_id,s1) values(2, 'd1', 'xyTestxy')", + "INSERT INTO trimTable(Time,device_id,s1) values(3, 'd1', ' Test ')", + "INSERT INTO trimTable(Time,device_id,s9) values(2, 'd1', 'xy')", + "INSERT INTO trimTable(Time,device_id,s9) values(3, 'd1', ' T')", + // upperSQL + "create table upperTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO upperTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 1, 1, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO upperTable(Time,device_id,s1) values(2, 'd1', 'Test')", + "INSERT INTO upperTable(Time,device_id,s1) values(3, 'd1', 'Abcdefg')", + "INSERT INTO upperTable(Time,device_id,s9) values(2, 'd1', 'Test')", + "INSERT INTO upperTable(Time,device_id,s9) values(3, 'd1', 'Abcdefg')", + // no args SQL + "create table NoArgTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO NoArgTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', 0, 0, 0, 0, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + // dateBinSQL use s8 to calculate + "create table dateBinTable(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + // 2024-01-01T00:00:00.000Z + "INSERT INTO dateBinTable(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'Test', 1, 1, 1, 1, true, '2024-01-01', 1704067200000, 'abcd', X'abcd')", + // 2024-01-01T01:00:00.000Z + "INSERT INTO dateBinTable(Time,device_id,s1,s8) values(2, 'd1', 'Test', 1704070800000)", + // 2024-01-01T01:59:00.000Z + "INSERT INTO dateBinTable(Time,device_id,s1,s8) values(3, 'd1', 'Test', 1704074340000)", + // 2023-12-31T23:59:00.000Z + "INSERT INTO dateBinTable(Time,device_id,s1,s8) values(4, 'd1', 'Test', 1704067140000)", + // 1969-12-31T23:59:00.000Z + "INSERT INTO dateBinTable(Time,device_id,s1,s8) values(5, 'd1', 'Test', -60000)", + // null + "INSERT INTO dateBinTable(Time,device_id,s1,s8) values(6, 'd1', 'Test', null)", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void absTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE + String[] expectedHeader = + new String[] {"time", "s2", "_col2", "s3", "_col4", "s4", "_col6", "s5", "_col8"}; + String[] expectedAns = + new String[] { + "1970-01-01T00:00:00.001Z,1,1,1,1,1.0,1.0,1.0,1.0,", + "1970-01-01T00:00:00.002Z,-1,1,-1,1,-1.5,1.5,-1.5,1.5,", + "1970-01-01T00:00:00.003Z,-2,2,-2,2,-2.5,2.5,-2.5,2.5,", + }; + tableResultSetEqualTest( + "select time,s2,abs(s2),s3,abs(s3),s4,abs(s4),s5,abs(s5) from absTable", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void testINT64NotIn() { + // case 1: support INT32, INT64, FLOAT, DOUBLE + String[] expectedHeader = new String[] {"time", "s3"}; + String[] expectedAns = + new String[] { + "1970-01-01T00:00:00.002Z,-1,", "1970-01-01T00:00:00.003Z,-2,", + }; + tableResultSetEqualTest( + "select time,s3 from absTable where s3 not in (1)", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void testBlobCompare() { + // case 1: support INT32, INT64, FLOAT, DOUBLE + String[] expectedHeader = new String[] {"s10", "res1", "res2", "res3"}; + String[] expectedAns = + new String[] { + "0xabcd,true,true,true,", "null,null,null,null,", "null,null,null,null,", + }; + tableResultSetEqualTest( + "select s10, s10 > x'2d' as res1, s10 <> x'2d' as res2, s10 = X'abcd' as res3 from absTable", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void testDateCompare() { + // case 1: support INT32, INT64, FLOAT, DOUBLE + String[] expectedHeader = new String[] {"s7", "res1", "res2", "res3"}; + String[] expectedAns = + new String[] { + "2021-10-01,true,true,true,", "null,null,null,null,", "null,null,null,null,", + }; + // add it back while supporting Implicit conversion + // tableResultSetEqualTest( + // "select s7, s7 < '2022-12-12' as res1, s7 <> '2022-12-12' as res2, s7 = '2021-10-01' + // as res3 from absTable", + // expectedHeader, + // expectedAns, + // DATABASE_NAME); + tableResultSetEqualTest( + "select s7, s7 < CAST('2022-12-12' AS DATE) as res1, s7 <> CAST('2022-12-12' AS DATE) AS res2, s7 = CAST('2021-10-01' AS DATE) as res3 from absTable", + expectedHeader, + expectedAns, + DATABASE_NAME); + + expectedHeader = new String[] {"s3", "s7"}; + expectedAns = + new String[] { + "1,2021-10-01,", + }; + + tableResultSetEqualTest( + "select s3, s7 from absTable where s7 in (CAST('2021-10-01' AS DATE), CAST('2021-10-02' AS DATE))", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void testTimestampCompare() { + // case 1: support INT32, INT64, FLOAT, DOUBLE + String[] expectedHeader = new String[] {"s2", "s8"}; + String[] expectedAns = + new String[] { + "1,2021-10-01T00:00:00.000Z,", + }; + + tableResultSetEqualTest( + "select s2, s8 from absTable where s8 IN (CAST('2021-10-01T08:00:00.000+08:00' AS TIMESTAMP), CAST('2021-10-01T00:00:00.000Z' AS TIMESTAMP))", + expectedHeader, + expectedAns, + DATABASE_NAME); + + tableResultSetEqualTest( + "select s2, s8 from absTable where s8=CAST('2021-10-01T08:00:00.000+08:00' AS TIMESTAMP)", + expectedHeader, + expectedAns, + DATABASE_NAME); + + tableResultSetEqualTest( + "select s2, s8 from absTable where s8=2021-10-01T08:00:00.000+08:00", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void absTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,abs(s2,1) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function abs only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,abs(s1) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function abs only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,abs(s6) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function abs only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,abs(s7) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function abs only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,abs(s9) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function abs only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,abs(s10) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function abs only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s7, s7 < '2022-12-12', s7 <> '2022-12-12', s7 = '2021-10-01' from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Cannot apply operator: DATE < STRING", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select CAST(s1 AS INT32) from absTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Cannot cast abcd to INT32 type", + DATABASE_NAME); + } + + @Test + public void acosTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP And range of input value is [-1, 1] + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.acos(1), Math.acos(2)}; + Double[] expectedResultLong = new Double[] {Math.acos(1), Math.acos(2)}; + Double[] expectedResultFloat = new Double[] {Math.acos(1.0f), Math.acos(0.5f)}; + Double[] expectedResultDouble = new Double[] {Math.acos(1.0), Math.acos(0.5)}; + testDoubleResult( + "select time,acos(s2),acos(s3),acos(s4),acos(s5) from acosTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void acosTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,acos(s2,1) from acosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function acos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,acos(s1) from acosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function acos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,acos(s6) from acosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function acos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,acos(s7) from acosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function acos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,acos(s9) from acosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function acos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,acos(s10) from acosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function acos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void asinTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP And range of input value is [-1, 1] + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.asin(1), Math.asin(2)}; + Double[] expectedResultLong = new Double[] {Math.asin(1), Math.asin(2)}; + Double[] expectedResultFloat = new Double[] {Math.asin(1.0f), Math.asin(0.5f)}; + Double[] expectedResultDouble = new Double[] {Math.asin(1.0), Math.asin(0.5)}; + testDoubleResult( + "select time,asin(s2),asin(s3),asin(s4),asin(s5) from asinTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void asinTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,asin(s2,1) from asinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function asin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,asin(s1) from asinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function asin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,asin(s6) from asinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function asin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,asin(s7) from asinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function asin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,asin(s9) from asinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function asin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,asin(s10) from asinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function asin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void atanTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.atan(1), Math.atan(2), Math.atan(3)}; + Double[] expectedResultLong = new Double[] {Math.atan(1), Math.atan(2), Math.atan(3)}; + Double[] expectedResultFloat = new Double[] {Math.atan(1.0f), Math.atan(2.5f), Math.atan(3.5f)}; + Double[] expectedResultDouble = new Double[] {Math.atan(1.0), Math.atan(2.5), Math.atan(3.5)}; + testDoubleResult( + "select time,atan(s2),atan(s3),atan(s4),atan(s5) from atanTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void atanTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,atan(s2,1) from atanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function atan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,atan(s1) from atanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function atan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,atan(s6) from atanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function atan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,atan(s7) from atanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function atan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,atan(s9) from atanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function atan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,atan(s10) from atanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function atan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void ceilTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.ceil(1), Math.ceil(2), Math.ceil(-2)}; + Double[] expectedResultLong = new Double[] {Math.ceil(1), Math.ceil(2), Math.ceil(-2)}; + Double[] expectedResultFloat = + new Double[] {Math.ceil(1.0f), Math.ceil(2.5f), Math.ceil(-2.5f)}; + Double[] expectedResultDouble = new Double[] {Math.ceil(1.0), Math.ceil(2.5), Math.ceil(-2.5)}; + testDoubleResult( + "select time,ceil(s2),ceil(s3),ceil(s4),ceil(s5) from ceilTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void ceilTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,ceil(s2,1) from ceilTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ceil only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,ceil(s1) from ceilTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ceil only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,ceil(s6) from ceilTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ceil only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,ceil(s7) from ceilTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ceil only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,ceil(s9) from ceilTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ceil only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,ceil(s10) from ceilTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ceil only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void concatTestNormal() { + // support the (measurement, ConstantArgument) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,abcdes,ab,abes,", + "1970-01-01T00:00:00.002Z,Test,Testes,Test,Testes,", + "1970-01-01T00:00:00.003Z,efgh,efghes,efgh,efghes,", + "1970-01-01T00:00:00.004Z,null,es,haha,hahaes,", + }; + tableResultSetEqualTest( + "select time,s1,concat(s1,'es'),s9,concat(s9,'es') from concatTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (ConstantArgument, measurement) + expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,esabcd,ab,esab,", + "1970-01-01T00:00:00.002Z,Test,esTest,Test,esTest,", + "1970-01-01T00:00:00.003Z,efgh,esefgh,efgh,esefgh,", + "1970-01-01T00:00:00.004Z,null,es,haha,eshaha,", + }; + tableResultSetEqualTest( + "select time,s1,concat('es',s1),s9,concat('es',s9) from concatTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (measurement, measurement) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,abcdab,", + "1970-01-01T00:00:00.002Z,Test,Test,TestTest,", + "1970-01-01T00:00:00.003Z,efgh,efgh,efghefgh,", + "1970-01-01T00:00:00.004Z,null,haha,haha,", + }; + tableResultSetEqualTest( + "select time,s1,s9,concat(s1,s9) from concatTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (string1,string2,string3...stringN) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,headerabcdbodyabtail,", + "1970-01-01T00:00:00.002Z,Test,Test,headerTestbodyTesttail,", + "1970-01-01T00:00:00.003Z,efgh,efgh,headerefghbodyefghtail,", + "1970-01-01T00:00:00.004Z,null,haha,headerbodyhahatail,", + }; + tableResultSetEqualTest( + "select time,s1,s9,concat('header',s1,'body',s9,'tail') from concatTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void concatTestFail() { + // case 1: less than two argument + tableAssertTestFail( + "select s1,concat(s1) from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s2,concat(s2, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s3,concat(s3, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s4,concat(s4, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s5,concat(s5, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s6,concat(s6, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s7,concat(s7, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s8,concat(s8, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s10,concat(s10, 'es') from concatTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function concat only accepts two or more arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void cosTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.cos(1), Math.cos(2), Math.cos(3)}; + Double[] expectedResultLong = new Double[] {Math.cos(1), Math.cos(2), Math.cos(3)}; + Double[] expectedResultFloat = new Double[] {Math.cos(1), Math.cos(2.5), Math.cos(3.5)}; + Double[] expectedResultDouble = new Double[] {Math.cos(1), Math.cos(2.5), Math.cos(3.5)}; + testDoubleResult( + "select time,cos(s2),cos(s3),cos(s4),cos(s5) from cosTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void cosTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,cos(s2,1) from cosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,cos(s1) from cosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,cos(s6) from cosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,cos(s7) from cosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,cos(s9) from cosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,cos(s10) from cosTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cos only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void coshTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.cosh(1), Math.cosh(2), Math.cosh(3)}; + Double[] expectedResultLong = new Double[] {Math.cosh(1), Math.cosh(2), Math.cosh(3)}; + Double[] expectedResultFloat = new Double[] {Math.cosh(1), Math.cosh(2.5), Math.cosh(3.5)}; + Double[] expectedResultDouble = new Double[] {Math.cosh(1), Math.cosh(2.5), Math.cosh(3.5)}; + testDoubleResult( + "select time,cosh(s2),cosh(s3),cosh(s4),cosh(s5) from coshTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void coshTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,cosh(s2,1) from coshTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cosh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,cosh(s1) from coshTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cosh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,cosh(s6) from coshTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cosh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,cosh(s7) from coshTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cosh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,cosh(s9) from coshTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cosh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,cosh(s10) from coshTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function cosh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void degreesTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = + new Double[] {Math.toDegrees(1), Math.toDegrees(2), Math.toDegrees(3)}; + Double[] expectedResultLong = + new Double[] {Math.toDegrees(1), Math.toDegrees(2), Math.toDegrees(3)}; + Double[] expectedResultFloat = + new Double[] {Math.toDegrees(1), Math.toDegrees(2.5), Math.toDegrees(3.5)}; + Double[] expectedResultDouble = + new Double[] {Math.toDegrees(1), Math.toDegrees(2.5), Math.toDegrees(3.5)}; + testDoubleResult( + "select time,degrees(s2),degrees(s3),degrees(s4),degrees(s5) from degreesTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void degreesTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,degrees(s2,1) from degreesTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function degrees only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,degrees(s1) from degreesTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function degrees only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,degrees(s6) from degreesTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function degrees only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,degrees(s7) from degreesTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function degrees only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,degrees(s9) from degreesTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function degrees only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,degrees(s10) from degreesTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function degrees only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void endsWithTestNormal() { + // support the (measurement, ConstantArgument) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,false,ab,false,", + "1970-01-01T00:00:00.002Z,Test,false,Test,false,", + "1970-01-01T00:00:00.003Z,efgh,true,efgh,true,", + }; + tableResultSetEqualTest( + "select time,s1,ends_with(s1,'gh'),s9,ends_with(s9,'gh') from endsWithTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (measurement, measurement) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,false,", + "1970-01-01T00:00:00.002Z,Test,Test,true,", + "1970-01-01T00:00:00.003Z,efgh,efgh,true,", + }; + tableResultSetEqualTest( + "select time,s1,s9,ends_with(s1,s9) from endsWithTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void endsWithTestFail() { + // case 1: more than two argument + tableAssertTestFail( + "select s1,ends_with(s1, 'es', 'ab') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: less than two argument + tableAssertTestFail( + "select s1,ends_with(s1) from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s2,ends_with(s2, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s3,ends_with(s3, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s4,ends_with(s4, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s5,ends_with(s5, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s6,ends_with(s6, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s7,ends_with(s7, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s8,ends_with(s8, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 10: wrong data type + tableAssertTestFail( + "select s10,ends_with(s10, 'es') from endsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ends_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void expTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.exp(1), Math.exp(2), Math.exp(3)}; + Double[] expectedResultLong = new Double[] {Math.exp(1), Math.exp(2), Math.exp(3)}; + Double[] expectedResultFloat = new Double[] {Math.exp(1), Math.exp(2.5), Math.exp(3.5)}; + Double[] expectedResultDouble = new Double[] {Math.exp(1), Math.exp(2.5), Math.exp(3.5)}; + testDoubleResult( + "select time,exp(s2),exp(s3),exp(s4),exp(s5) from expTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void expTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,exp(s2,1) from expTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function exp only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,exp(s1) from expTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function exp only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,exp(s6) from expTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function exp only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,exp(s7) from expTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function exp only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,exp(s9) from expTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function exp only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,exp(s10) from expTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function exp only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void floorTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.floor(1), Math.floor(2), Math.floor(-2)}; + Double[] expectedResultLong = new Double[] {Math.floor(1), Math.floor(2), Math.floor(-2)}; + Double[] expectedResultFloat = + new Double[] {Math.floor(1.0f), Math.floor(2.5f), Math.floor(-2.5f)}; + Double[] expectedResultDouble = + new Double[] {Math.floor(1.0), Math.floor(2.5), Math.floor(-2.5)}; + testDoubleResult( + "select time,floor(s2),floor(s3),floor(s4),floor(s5) from floorTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void floorTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,floor(s2,1) from floorTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function floor only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,floor(s1) from floorTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function floor only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,floor(s6) from floorTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function floor only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,floor(s7) from floorTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function floor only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,floor(s9) from floorTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function floor only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,floor(s10) from floorTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function floor only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void lengthTestNormal() { + // case 1: support Text data type + String[] expectedHeader = new String[] {"time", "s1", "_col2"}; + String[] expectedAns = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,4,", + "1970-01-01T00:00:00.002Z,test,4,", + "1970-01-01T00:00:00.003Z,abcdefg,7,", + }; + tableResultSetEqualTest( + "select time,s1,Length(s1) from lengthTable", expectedHeader, expectedAns, DATABASE_NAME); + + // case 2: support String data type + expectedHeader = new String[] {"time", "s9", "_col2"}; + expectedAns = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,4,", + "1970-01-01T00:00:00.002Z,test,4,", + "1970-01-01T00:00:00.003Z,abcdefg,7,", + }; + tableResultSetEqualTest( + "select time,s9,Length(s9) from lengthTable", expectedHeader, expectedAns, DATABASE_NAME); + } + + @Test + public void lengthTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s1,Length(s1,1) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,Length(s2) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s1,Length(s3) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s1,Length(s4) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s1,Length(s5) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s1,Length(s6) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s1,Length(s7) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s1,Length(s8) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s1,Length(s10) from lengthTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void lnTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.log(1), Math.log(0), Math.log(-3)}; + Double[] expectedResultLong = new Double[] {Math.log(1), Math.log(0), Math.log(-3)}; + Double[] expectedResultFloat = new Double[] {Math.log(1), Math.log(0), Math.log(-3.5)}; + Double[] expectedResultDouble = new Double[] {Math.log(1), Math.log(0), Math.log(-3.5)}; + testDoubleResult( + "select time,ln(s2),ln(s3),ln(s4),ln(s5) from lnTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void lnTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,ln(s2,1) from lnTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ln only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,ln(s1) from lnTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ln only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,ln(s6) from lnTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ln only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,ln(s7) from lnTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ln only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,ln(s9) from lnTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ln only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,ln(s10) from lnTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function ln only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void log10TestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.log10(1), Math.log10(0), Math.log10(-3)}; + Double[] expectedResultLong = new Double[] {Math.log10(1), Math.log10(0), Math.log10(-3)}; + Double[] expectedResultFloat = new Double[] {Math.log10(1), Math.log10(0), Math.log10(-3.5)}; + Double[] expectedResultDouble = new Double[] {Math.log10(1), Math.log10(0), Math.log10(-3.5)}; + testDoubleResult( + "select time,log10(s2),log10(s3),log10(s4),log10(s5) from log10Table", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void log10TestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,log10(s2,1) from log10Table", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function log10 only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,log10(s1) from log10Table", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function log10 only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,log10(s6) from log10Table", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function log10 only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,log10(s7) from log10Table", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function log10 only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,log10(s9) from log10Table", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function log10 only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,log10(s10) from log10Table", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function log10 only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void lowerTestNormal() { + // Normal + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,ABCD,abcd,ABCD,abcd,", + "1970-01-01T00:00:00.002Z,Test,test,Test,test,", + "1970-01-01T00:00:00.003Z,Abcdefg,abcdefg,Abcdefg,abcdefg,", + }; + tableResultSetEqualTest( + "select time,s1,lower(s1),s9,lower(s9) from lowerTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lowerTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s1,lower(s1, 1) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s2,lower(s2) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s3,lower(s3) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s4,lower(s4) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s5,lower(s5) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s6,lower(s6) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s7,lower(s7) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s8,lower(s8) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s10,lower(s10) from lowerTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function lower only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void radiansTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = + new Double[] {Math.toRadians(1), Math.toRadians(2), Math.toRadians(3)}; + Double[] expectedResultLong = + new Double[] {Math.toRadians(1), Math.toRadians(2), Math.toRadians(3)}; + Double[] expectedResultFloat = + new Double[] {Math.toRadians(1), Math.toRadians(2.5), Math.toRadians(3.5)}; + Double[] expectedResultDouble = + new Double[] {Math.toRadians(1), Math.toRadians(2.5), Math.toRadians(3.5)}; + testDoubleResult( + "select time,radians(s2),radians(s3),radians(s4),radians(s5) from radiansTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + private void testDoubleResult( + String sql, + String[] expectedHeader, + String database, + Double[] expectedResultInt, + Double[] expectedResultLong, + Double[] expectedResultFloat, + Double[] expectedResultDouble) { + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + assertEquals( + expectedResultInt[cnt], Double.parseDouble(resultSet.getString(2)), 0.00001); + assertEquals( + expectedResultLong[cnt], Double.parseDouble(resultSet.getString(3)), 0.00001); + assertEquals( + expectedResultFloat[cnt], Double.parseDouble(resultSet.getString(4)), 0.00001); + assertEquals( + expectedResultDouble[cnt], Double.parseDouble(resultSet.getString(5)), 0.00001); + cnt++; + } + assertEquals(expectedResultInt.length, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void radiansTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,radians(s2,1) from radiansTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function radians only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,radians(s1) from radiansTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function radians only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,radians(s6) from radiansTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function radians only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,radians(s7) from radiansTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function radians only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,radians(s9) from radiansTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function radians only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,radians(s10) from radiansTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function radians only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void regexpLikeTestNormal() { + // support the (measurement, ConstantArgument) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,true,ab,true,", + "1970-01-01T00:00:00.002Z,Test,false,Test,false,", + "1970-01-01T00:00:00.003Z,efgh,false,[e-g]+,false,", + }; + tableResultSetEqualTest( + "select time,s1,regexp_like(s1,'^abcd$'),s9,regexp_like(s9,'[a-h]+') from regexpLikeTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (measurement, measurement) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,false,", + "1970-01-01T00:00:00.002Z,Test,Test,true,", + "1970-01-01T00:00:00.003Z,efgh,[e-g]+,false,", + }; + tableResultSetEqualTest( + "select time,s1,s9,regexp_like(s1,s9) from regexpLikeTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void regexpLikeTestFail() { + // case 1: more than two argument + tableAssertTestFail( + "select s1,regexp_like(s1, 'es', 'ab') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: less than two argument + tableAssertTestFail( + "select s1,regexp_like(s1) from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s2,regexp_like(s2, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s3,regexp_like(s3, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s4,regexp_like(s4, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s5,regexp_like(s5, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s6,regexp_like(s6, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s7,regexp_like(s7, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s8,regexp_like(s8, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 10: wrong data type + tableAssertTestFail( + "select s10,regexp_like(s10, 'es') from regexpLikeTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function regexp_like only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void signTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = + new String[] {"time", "s2", "_col2", "s3", "_col4", "s4", "_col6", "s5", "_col8"}; + String[] expectedAns = + new String[] { + "1970-01-01T00:00:00.001Z,1,1,1,1,1.0,1.0,1.0,1.0,", + "1970-01-01T00:00:00.002Z,0,0,0,0,0.0,0.0,0.0,0.0,", + "1970-01-01T00:00:00.003Z,-1,-1,-1,-1,-1.0,-1.0,-1.0,-1.0,", + }; + tableResultSetEqualTest( + "select time,s2,sign(s2),s3,sign(s3),s4,sign(s4),s5,sign(s5) from signTable", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void signTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,sign(s2,1) from signTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sign only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,sign(s1) from signTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sign only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,sign(s6) from signTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sign only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,sign(s7) from signTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sign only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,sign(s9) from signTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sign only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,sign(s10) from signTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sign only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void sinTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.sin(1), Math.sin(2), Math.sin(3)}; + Double[] expectedResultLong = new Double[] {Math.sin(1), Math.sin(2), Math.sin(3)}; + Double[] expectedResultFloat = new Double[] {Math.sin(1), Math.sin(2.5), Math.sin(3.5)}; + Double[] expectedResultDouble = new Double[] {Math.sin(1), Math.sin(2.5), Math.sin(3.5)}; + testDoubleResult( + "select time,sin(s2),sin(s3),sin(s4),sin(s5) from sinTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void sinTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,sin(s2,1) from sinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,sin(s1) from sinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,sin(s6) from sinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,sin(s7) from sinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,sin(s9) from sinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,sin(s10) from sinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sin only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void sinhTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.sinh(1), Math.sinh(2), Math.sinh(3)}; + Double[] expectedResultLong = new Double[] {Math.sinh(1), Math.sinh(2), Math.sinh(3)}; + Double[] expectedResultFloat = new Double[] {Math.sinh(1), Math.sinh(2.5), Math.sinh(3.5)}; + Double[] expectedResultDouble = new Double[] {Math.sinh(1), Math.sinh(2.5), Math.sinh(3.5)}; + testDoubleResult( + "select time,sinh(s2),sinh(s3),sinh(s4),sinh(s5) from sinhTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void sinhTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,sinh(s2,1) from sinhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sinh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,sinh(s1) from sinhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sinh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,sinh(s6) from sinhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sinh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,sinh(s7) from sinhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sinh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,sinh(s9) from sinhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sinh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,sinh(s10) from sinhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sinh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void sqrtTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.sqrt(1), Math.sqrt(0), Math.sqrt(-1)}; + Double[] expectedResultLong = new Double[] {Math.sqrt(1), Math.sqrt(0), Math.sqrt(-1)}; + Double[] expectedResultFloat = new Double[] {Math.sqrt(1), Math.sqrt(0), Math.sqrt(-1.5)}; + Double[] expectedResultDouble = new Double[] {Math.sqrt(1), Math.sqrt(0), Math.sqrt(-1.5)}; + testDoubleResult( + "select time,sqrt(s2),sqrt(s3),sqrt(s4),sqrt(s5) from sqrtTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void sqrtTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,sqrt(s2,1) from sqrtTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sqrt only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,sqrt(s1) from sqrtTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sqrt only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,sqrt(s6) from sqrtTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sqrt only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,sqrt(s7) from sqrtTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sqrt only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,sqrt(s9) from sqrtTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sqrt only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,sqrt(s10) from sqrtTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function sqrt only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void startsWithTestNormal() { + // support the (measurement, ConstantArgument) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,false,ab,false,", + "1970-01-01T00:00:00.002Z,Test,true,Test,true,", + "1970-01-01T00:00:00.003Z,efgh,false,efgh,false,", + }; + tableResultSetEqualTest( + "select time,s1,starts_with(s1,'Te'),s9,starts_with(s9,'Te') from startsWithTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (measurement, measurement) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,true,", + "1970-01-01T00:00:00.002Z,Test,Test,true,", + "1970-01-01T00:00:00.003Z,efgh,efgh,true,", + }; + tableResultSetEqualTest( + "select time,s1,s9,starts_with(s1,s9) from startsWithTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void startsWithTestFail() { + // case 1: more than two argument + tableAssertTestFail( + "select s1,starts_with(s1, 'es', 'ab') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: less than two argument + tableAssertTestFail( + "select s1,starts_with(s1) from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s2,starts_with(s2, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s3,starts_with(s3, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s4,starts_with(s4, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s5,starts_with(s5, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s6,starts_with(s6, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s7,starts_with(s7, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s8,starts_with(s8, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 10: wrong data type + tableAssertTestFail( + "select s10,starts_with(s10, 'es') from startsWithTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function starts_with only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void strcmpTestNormal() { + // support the (measurement, ConstantArgument) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,-1,ab,0,", + "1970-01-01T00:00:00.002Z,Test,-1,Test,-1,", + "1970-01-01T00:00:00.003Z,efgh,0,efgh,1,", + }; + tableResultSetEqualTest( + "select time,s1,strcmp(s1,'efgh'),s9,strcmp(s9,'ab') from strcmpTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (measurement, measurement) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,1,", + "1970-01-01T00:00:00.002Z,Test,Test,0,", + "1970-01-01T00:00:00.003Z,efgh,efgh,0,", + }; + tableResultSetEqualTest( + "select time,s1,s9,strcmp(s1,s9) from strcmpTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void strcmpTestFail() { + // case 1: more than two argument + tableAssertTestFail( + "select s1,strcmp(s1, 'es', 'ab') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: less than two argument + tableAssertTestFail( + "select s1,strcmp(s1) from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s2,strcmp(s2, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s3,strcmp(s3, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s4,strcmp(s4, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s5,strcmp(s5, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s6,strcmp(s6, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s7,strcmp(s7, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s8,strcmp(s8, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 10: wrong data type + tableAssertTestFail( + "select s10,strcmp(s10, 'es') from strcmpTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strcmp only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void strposTestNormal() { + // support the (measurement, ConstantArgument) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,0,ab,0,", + "1970-01-01T00:00:00.002Z,Test,2,Test,2,", + "1970-01-01T00:00:00.003Z,efgh,0,efgh,0,", + }; + tableResultSetEqualTest( + "select time,s1,strpos(s1,'es'),s9,strpos(s9,'es') from strposTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the (measurement, measurement) + expectedHeader = new String[] {"time", "s1", "s9", "_col3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,1,", + "1970-01-01T00:00:00.002Z,Test,Test,1,", + "1970-01-01T00:00:00.003Z,efgh,efgh,1,", + }; + tableResultSetEqualTest( + "select time,s1,s9,strpos(s1,s9) from strposTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void strposTestFail() { + // case 1: more than two argument + tableAssertTestFail( + "select s1,strpos(s1, 'es', 'ab') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: less than two argument + tableAssertTestFail( + "select s1,strpos(s1) from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s2,strpos(s2, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s3,strpos(s3, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s4,strpos(s4, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s5,strpos(s5, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s6,strpos(s6, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s7,strpos(s7, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s8,strpos(s8, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 10: wrong data type + tableAssertTestFail( + "select s10,strpos(s10, 'es') from strposTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function strpos only accepts two arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void tanTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.tan(1), Math.tan(2), Math.tan(3)}; + Double[] expectedResultLong = new Double[] {Math.tan(1), Math.tan(2), Math.tan(3)}; + Double[] expectedResultFloat = + new Double[] {Math.tan(1), Math.tan((float) 1.57079632675), Math.tan(3.5)}; + Double[] expectedResultDouble = + new Double[] {Math.tan(1), Math.tan(1.57079632675), Math.tan(3.5)}; + testDoubleResult( + "select time,tan(s2),tan(s3),tan(s4),tan(s5) from tanTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void tanTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,tan(s2,1) from tanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,tan(s1) from tanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,tan(s6) from tanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,tan(s7) from tanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,tan(s9) from tanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,tan(s10) from tanTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tan only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void tanhTestNormal() { + // case 1: support INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.tanh(1), Math.tanh(2), Math.tanh(3)}; + Double[] expectedResultLong = new Double[] {Math.tanh(1), Math.tanh(2), Math.tanh(3)}; + Double[] expectedResultFloat = new Double[] {Math.tanh(1), Math.tanh(2.5), Math.tanh(3.5)}; + Double[] expectedResultDouble = new Double[] {Math.tanh(1), Math.tanh(2.5), Math.tanh(3.5)}; + testDoubleResult( + "select time,tanh(s2),tanh(s3),tanh(s4),tanh(s5) from tanhTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void tanhTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,tanh(s2,1) from tanhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tanh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s1,tanh(s1) from tanhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tanh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s6,tanh(s6) from tanhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tanh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s7,tanh(s7) from tanhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tanh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s9,tanh(s9) from tanhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tanh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s10,tanh(s10) from tanhTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function tanh only accepts one argument and it must be Double, Float, Int32 or Int64 data type.", + DATABASE_NAME); + } + + @Test + public void trimTestNormal() { + // support the trim(trimSource) trim(specification From trimSource) + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,abcd,ab,ab,", + "1970-01-01T00:00:00.002Z,xyTestxy,xyTestxy,xy,xy,", + "1970-01-01T00:00:00.003Z, Test ,Test , T,T,", + }; + tableResultSetEqualTest( + "select time,s1,trim(LEADING FROM s1),s9,trim(s9) from trimTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the trim(trimSource, trimChar) trim(trimChar From trimSource) + expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,bc,ab,b,", + "1970-01-01T00:00:00.002Z,xyTestxy,yTestxy,xy,y,", + "1970-01-01T00:00:00.003Z, Test , Test , T, T,", + }; + tableResultSetEqualTest( + "select time,s1,trim(s1, 'axd'),s9,trim('ax' FROM s9) from trimTable", + expectedHeader, + retArray, + DATABASE_NAME); + + // support the trim(trimSpecification trimChar From trimSource) + expectedHeader = new String[] {"time", "s1", "s9", "_col3", "_col4"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ab,cd,abcd,", + "1970-01-01T00:00:00.002Z,xyTestxy,xy,Test,xyTest,", + "1970-01-01T00:00:00.003Z, Test , T,est, Test,", + }; + tableResultSetEqualTest( + "select time,s1,s9,trim(BOTH s9 FROM s1), trim(TRAILING s9 FROM s1)from trimTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void trimTestFail() { + // case 1: wrong data type + tableAssertTestFail( + "select s2,trim(s2, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s3,trim(s3, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s4,trim(s4, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s5,trim(s5, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s6,trim(s6, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s7,trim(s7, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s8,trim(s8, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s10,trim(s10, 'es') from trimTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function trim only accepts one or two arguments and they must be text or string data type.", + DATABASE_NAME); + } + + @Test + public void upperTestNormal() { + // Normal + String[] expectedHeader = new String[] {"time", "s1", "_col2", "s9", "_col4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,ABCD,abcd,ABCD,", + "1970-01-01T00:00:00.002Z,Test,TEST,Test,TEST,", + "1970-01-01T00:00:00.003Z,Abcdefg,ABCDEFG,Abcdefg,ABCDEFG,", + }; + tableResultSetEqualTest( + "select time,s1,upper(s1),s9,upper(s9) from upperTable", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void upperTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s1,upper(s1, 1) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 2: wrong data type + tableAssertTestFail( + "select s2,upper(s2) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 3: wrong data type + tableAssertTestFail( + "select s3,upper(s3) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 4: wrong data type + tableAssertTestFail( + "select s4,upper(s4) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 5: wrong data type + tableAssertTestFail( + "select s5,upper(s5) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 6: wrong data type + tableAssertTestFail( + "select s6,upper(s6) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 7: wrong data type + tableAssertTestFail( + "select s7,upper(s7) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 8: wrong data type + tableAssertTestFail( + "select s8,upper(s8) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + + // case 9: wrong data type + tableAssertTestFail( + "select s10,upper(s10) from upperTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function upper only accepts one argument and it must be text or string data type.", + DATABASE_NAME); + } + + private void testOneRowDoubleResult( + String sql, String[] expectedHeader, String database, Double[] expectedResult) { + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + resultSet.next(); + assertEquals(expectedResult[0], Double.parseDouble(resultSet.getString(1)), 0.00001); + assertEquals(expectedResult[1], Double.parseDouble(resultSet.getString(2)), 0.00001); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void piTestNormal() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.PI}; + Double[] expectedResultLong = new Double[] {Math.PI}; + Double[] expectedResultFloat = new Double[] {Math.PI}; + Double[] expectedResultDouble = new Double[] {Math.PI}; + testDoubleResult( + "select time, s2 + pi(), s3 + pi(), s4 + pi(), s5 + pi() from NoArgTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void piTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s2,pi(s2) from NoArgTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Scalar function pi accepts no argument.", + DATABASE_NAME); + } + + @Test + public void eTestNormal() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2", "_col3", "_col4"}; + Double[] expectedResultInt = new Double[] {Math.E}; + Double[] expectedResultLong = new Double[] {Math.E}; + Double[] expectedResultFloat = new Double[] {Math.E}; + Double[] expectedResultDouble = new Double[] {Math.E}; + testDoubleResult( + "select time, s2 + e(), s3 + e(), s4 + e(), s5 + e() from NoArgTable", + expectedHeader, + DATABASE_NAME, + expectedResultInt, + expectedResultLong, + expectedResultFloat, + expectedResultDouble); + } + + @Test + public void eTestFail() { + // case 1: more than one argument + tableAssertTestFail( + "select s1,e(s1) from NoArgTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Scalar function e accepts no argument.", + DATABASE_NAME); + } + + @Test + public void dateBinTestNormal() { + String[] expectedHeader = new String[] {"time", "s1", "s8", "_col3"}; + String[] expectedAns = + new String[] { + "1970-01-01T00:00:00.001Z,Test,2024-01-01T00:00:00.000Z,2024-01-01T00:00:00.000Z,", + "1970-01-01T00:00:00.002Z,Test,2024-01-01T01:00:00.000Z,2024-01-01T01:00:00.000Z,", + "1970-01-01T00:00:00.003Z,Test,2024-01-01T01:59:00.000Z,2024-01-01T01:00:00.000Z,", + "1970-01-01T00:00:00.004Z,Test,2023-12-31T23:59:00.000Z,2023-12-31T23:00:00.000Z,", + "1970-01-01T00:00:00.005Z,Test,1969-12-31T23:59:00.000Z,1969-12-31T23:00:00.000Z,", + "1970-01-01T00:00:00.006Z,Test,null,null,", + }; + tableResultSetEqualTest( + "select time,s1,s8,date_bin(1H, s8) from dateBinTable", + expectedHeader, + expectedAns, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "s8", "_col2"}; + expectedAns = + new String[] { + "1970-01-01T00:00:00.001Z,2024-01-01T00:00:00.000Z,2023-12-28T00:00:00.000Z,", + "1970-01-01T00:00:00.002Z,2024-01-01T01:00:00.000Z,2023-12-28T00:00:00.000Z,", + "1970-01-01T00:00:00.003Z,2024-01-01T01:59:00.000Z,2023-12-28T00:00:00.000Z,", + "1970-01-01T00:00:00.004Z,2023-12-31T23:59:00.000Z,2023-12-28T00:00:00.000Z,", + "1970-01-01T00:00:00.005Z,1969-12-31T23:59:00.000Z,1969-12-25T00:00:00.000Z,", + "1970-01-01T00:00:00.006Z,null,null,", + }; + tableResultSetEqualTest( + "select time,s8,date_bin(1W, s8) from dateBinTable", + expectedHeader, + expectedAns, + DATABASE_NAME); + } + + @Test + public void dateBinTestFail() { + tableAssertTestFail( + "select time,s1,s8,date_bin(1H,s8,0,0) from dateBinTable", + TSStatusCode.SQL_PARSE_ERROR.getStatusCode() + + ": line 1:35: mismatched input ','. Expecting: ')'", + DATABASE_NAME); + + tableAssertTestFail( + "select time,s1,s8,date_bin(1H,s1) from dateBinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function date_bin only accepts two or three arguments and the second and third must be TimeStamp data type.", + DATABASE_NAME); + + tableAssertTestFail( + "select time,s1,s8,date_bin(1MONTH 1DAY,s8) from dateBinTable", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Simultaneous setting of monthly and non-monthly intervals is not supported.", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBSubStringFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBSubStringFunctionTableIT.java new file mode 100644 index 0000000000000..4cd6d3f5167e8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBSubStringFunctionTableIT.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSubStringFunctionTableIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create table table1(device_id STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD, s3 INT64 FIELD, s4 FLOAT FIELD, s5 DOUBLE FIELD, s6 BOOLEAN FIELD, s7 DATE FIELD, s8 TIMESTAMP FIELD, s9 STRING FIELD, s10 BLOB FIELD)", + "INSERT INTO table1(Time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1, 'd1', 'abcd', -1, 3, 1, 1, true, '2021-10-01', 1633046400000, 'abcd', X'abcd')", + "INSERT INTO table1(Time,device_id,s1) values(2, 'd1', 'test')", + "INSERT INTO table1(Time,device_id,s1) values(3, 'd1', 'abcdefg')", + "INSERT INTO table1(Time,device_id,s9) values(2, 'd1', 'test')", + "INSERT INTO table1(Time,device_id,s9) values(3, 'd1', 'abcdefg')", + "INSERT INTO table1(Time,device_id,s2,s3) values(2, 'd1', -1, 10)", + "INSERT INTO table1(Time,device_id,s2,s3) values(3, 'd1', 2, 3)", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testNewTransformer() { + // Normal + String[] expectedHeader = new String[] {"time", "s1", "_col2", "_col3", "s9", "_col5", "_col6"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,abcd,abc,abcd,abcd,abc,", + "1970-01-01T00:00:00.002Z,test,test,tes,test,test,tes,", + "1970-01-01T00:00:00.003Z,abcdefg,abcdefg,abc,abcdefg,abcdefg,abc,", + }; + tableResultSetEqualTest( + "select time,s1,SUBSTRING(s1 FROM 1),SUBSTRING(s1 FROM 1 FOR 3),s9,SUBSTRING(s9 from 1),SUBSTRING(s9 from 1 for 3) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2 column + expectedHeader = new String[] {"time", "s1", "s9", "s2", "_col4", "_col5"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,abcd,-1,abcd,abcd,", + "1970-01-01T00:00:00.002Z,test,test,-1,test,test,", + "1970-01-01T00:00:00.003Z,abcdefg,abcdefg,2,bcdefg,bcdefg,", + }; + tableResultSetEqualTest( + "select time,s1,s9,s2,SUBSTRING(s1 from s2),SUBSTRING(s1 from s2) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3 column + expectedHeader = new String[] {"time", "s1", "s9", "s2", "s3", "_col5", "_col6"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,abcd,abcd,-1,3,a,a,", + "1970-01-01T00:00:00.002Z,test,test,-1,10,test,test,", + "1970-01-01T00:00:00.003Z,abcdefg,abcdefg,2,3,bcd,bcd,", + }; + tableResultSetEqualTest( + "select time,s1,s9,s2,s3,SUBSTRING(s1 from s2 for s3),SUBSTRING(s1 from s2 for s3) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testRoundBooleanAndText() { + // Using substring without start and end position. + tableAssertTestFail( + "select s1,SUBSTRING(s1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s2 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s3 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s4 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s5 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s6 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s7 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s8 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Wrong input type + tableAssertTestFail( + "select SUBSTRING(s10 FROM 1 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Using substring with float start position + tableAssertTestFail( + "select SUBSTRING(s1 FROM 1.0 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Using substring with float start and length + tableAssertTestFail( + "select SUBSTRING(s1 FROM 1.0 FOR 1.1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function substring only accepts two or three arguments and first must be text or string data type, second and third must be numeric data types [INT32, INT64]", + DATABASE_NAME); + + // Negative characters length + tableAssertTestFail( + "select SUBSTRING(s1 FROM 1 FOR -10) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Argument exception,the scalar function substring length must not be less than 0", + DATABASE_NAME); + + // big characters begin + tableAssertTestFail( + "select SUBSTRING(s1 FROM 100 FOR 1) from table1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Argument exception,the scalar function substring beginPosition must not be greater than the string length", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBOrderByTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBOrderByTableIT.java new file mode 100644 index 0000000000000..34f654d80e16d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBOrderByTableIT.java @@ -0,0 +1,1575 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.orderBy; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static org.apache.iotdb.relational.it.query.old.aligned.TableUtils.USE_DB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByTableIT { + + // the data can be viewed in + // https://docs.google.com/spreadsheets/d/1OWA1bKraArCwWVnuTjuhJ5yLG0PFLdD78gD6FjquepI/edit#gid=0 + private static final String[] sql1 = + new String[] { + "CREATE DATABASE db", + "USE db", + "CREATE TABLE table0 (device string tag, attr1 string attribute, num int32 field, bigNum int64 field, " + + "floatNum double field, str TEXT field, bool BOOLEAN field)", + "insert into table0(device, attr1, time,num,bigNum,floatNum,str,bool) values('d1', 'high', 0,3,2947483648,231.2121,'coconut',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 20,2,2147483648,434.12,'pineapple',TRUE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 40,1,2247483648,12.123,'apricot',TRUE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 80,9,2147483646,43.12,'apple',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 100,8,2147483964,4654.231,'papaya',TRUE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 31536000000,6,2147483650,1231.21,'banana',TRUE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 31536000100,10,3147483648,231.55,'pumelo',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 31536000500,4,2147493648,213.1,'peach',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 31536001000,5,2149783648,56.32,'orange',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 31536010000,7,2147983648,213.112,'lemon',TRUE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 31536100000,11,2147468648,54.121,'pitaya',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 41536000000,12,2146483648,45.231,'strawberry',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 41536000020,14,2907483648,231.34,'cherry',FALSE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 41536900000,13,2107483648,54.12,'lychee',TRUE)", + "insert into table0(device, time,num,bigNum,floatNum,str,bool) values('d1', 51536000000,15,3147483648,235.213,'watermelon',TRUE)" + }; + + private static final String[] sql2 = + new String[] { + "insert into table0(device,attr1,time,num,bigNum,floatNum,str,bool) values('d2','high',0,3,2947483648,231.2121,'coconut',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',20,2,2147483648,434.12,'pineapple',TRUE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',40,1,2247483648,12.123,'apricot',TRUE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',80,9,2147483646,43.12,'apple',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',100,8,2147483964,4654.231,'papaya',TRUE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',31536000000,6,2147483650,1231.21,'banana',TRUE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',31536000100,10,3147483648,231.55,'pumelo',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',31536000500,4,2147493648,213.1,'peach',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',31536001000,5,2149783648,56.32,'orange',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',31536010000,7,2147983648,213.112,'lemon',TRUE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',31536100000,11,2147468648,54.121,'pitaya',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',41536000000,12,2146483648,45.231,'strawberry',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',41536000020,14,2907483648,231.34,'cherry',FALSE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',41536900000,13,2107483648,54.12,'lychee',TRUE)", + "insert into table0(device,time,num,bigNum,floatNum,str,bool) values('d2',51536000000,15,3147483648,235.213,'watermelon',TRUE)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sql1) { + statement.execute(sql); + } + for (String sql : sql2) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + // sessionInsertData1(); + // sessionInsertData2(); + } + + // ordinal data + public static final String[][] RES = + new String[][] { + {"0", "3", "2947483648", "231.2121", "coconut", "false"}, + {"20", "2", "2147483648", "434.12", "pineapple", "true"}, + {"40", "1", "2247483648", "12.123", "apricot", "true"}, + {"80", "9", "2147483646", "43.12", "apple", "false"}, + {"100", "8", "2147483964", "4654.231", "papaya", "true"}, + {"31536000000", "6", "2147483650", "1231.21", "banana", "true"}, + {"31536000100", "10", "3147483648", "231.55", "pumelo", "false"}, + {"31536000500", "4", "2147493648", "213.1", "peach", "false"}, + {"31536001000", "5", "2149783648", "56.32", "orange", "false"}, + {"31536010000", "7", "2147983648", "213.112", "lemon", "true"}, + {"31536100000", "11", "2147468648", "54.121", "pitaya", "false"}, + {"41536000000", "12", "2146483648", "45.231", "strawberry", "false"}, + {"41536000020", "14", "2907483648", "231.34", "cherry", "false"}, + {"41536900000", "13", "2107483648", "54.12", "lychee", "true"}, + {"51536000000", "15", "3147483648", "235.213", "watermelon", "true"}, + }; + + private void checkHeader(ResultSetMetaData resultSetMetaData, String[] title) + throws SQLException { + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(title[i - 1], resultSetMetaData.getColumnName(i)); + } + } + + private void testNormalOrderBy(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualNum = resultSet.getString(2); + String actualBigNum = resultSet.getString(3); + double actualFloatNum = resultSet.getDouble(4); + String actualStr = resultSet.getString(5); + String actualBool = resultSet.getString(6); + + // System.out.println(actualTime + "," + actualNum + "," + actualBigNum + "," + + // actualFloatNum); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void orderByTest1() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 as t where t.device='d1' order by t.num"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest2() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 t where t.device='d1' order by bigNum,t.time"; + int[] ans = {13, 11, 10, 3, 1, 5, 4, 7, 9, 8, 2, 12, 0, 6, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest3() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by floatNum"; + int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest4() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest5() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by num desc"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + ArrayUtils.reverse(ans); + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest6() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bigNum desc, time asc"; + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest7() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by floatNum desc"; + int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; + ArrayUtils.reverse(ans); + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest8() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str desc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + ArrayUtils.reverse(ans); + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest15() { + String sql = "select Time,num+bigNum,floatNum from table0 where device='d1' order by str"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "_col1", "floatNum"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + double actualNum = resultSet.getLong(2); + double actualFloat = resultSet.getDouble(3); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Long.parseLong(RES[ans[i]][2]), actualNum, 0.0001); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloat, 0.0001); + + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + // 2. Multi-level order by test + @Test + public void orderByTest9() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bool asc, str asc"; + int[] ans = {3, 12, 0, 8, 7, 10, 6, 11, 2, 5, 9, 13, 4, 1, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest10() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bool asc, num desc"; + int[] ans = {12, 11, 10, 6, 3, 8, 7, 0, 14, 13, 4, 9, 5, 1, 2}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest11() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bigNum desc, floatNum desc"; + int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest12() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str desc, floatNum desc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void orderByTest13() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by num+floatNum desc, floatNum desc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest14() { + String sql = + "select Time,num+bigNum from table0 where device='d1' order by num+floatNum desc, floatNum desc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "_col1"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + double actualNum = resultSet.getLong(2); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Long.parseLong(RES[ans[i]][2]), actualNum, 0.001); + + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void orderByTest16() { + String sql = + "select Time,num+floatNum from table0 where device='d1' order by floatNum+num desc, floatNum desc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "_col1"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + double actualNum = resultSet.getDouble(2); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Double.parseDouble(RES[ans[i]][3]), + actualNum, + 0.001); + + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void orderByTest17() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str desc, str asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void orderByTest18() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str, str"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, ans); + } + + // limit cannot be pushed down in ORDER BY + @Test + public void orderByTest19() { + String sql = "select Time,num from table0 where device='d1' order by num limit 5"; + int[] ans = {2, 1, 0, 7, 8}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "num"}); + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + String actualNum = resultSet.getString(2); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + // 3. aggregation query + @Ignore + @Test + public void orderByInAggregationTest() { + String sql = "select avg(num) from root.sg.d group by session(10000ms) order by avg(num) desc"; + double[][] ans = new double[][] {{15.0}, {13.0}, {13.0}, {11.0}, {6.4}, {4.6}}; + long[] times = + new long[] {51536000000L, 41536000000L, 41536900000L, 31536100000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest2() { + String sql = + "select avg(num) from root.sg.d group by session(10000ms) order by max_value(floatNum)"; + double[][] ans = + new double[][] { + {13.0, 54.12}, + {11.0, 54.121}, + {13.0, 231.34}, + {15.0, 235.213}, + {6.4, 1231.21}, + {4.6, 4654.231} + }; + long[] times = + new long[] {41536900000L, 31536100000L, 41536000000L, 51536000000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest3() { + String sql = + "select avg(num) from root.sg.d group by session(10000ms) order by avg(num) desc,max_value(floatNum)"; + double[] ans = new double[] {15.0, 13.0, 13.0, 11.0, 6.4, 4.6}; + long[] times = + new long[] {51536000000L, 41536900000L, 41536000000L, 31536100000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest4() { + String sql = + "select avg(num)+avg(floatNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; + double[][] ans = + new double[][] {{1079.56122}, {395.4584}, {65.121}, {151.2855}, {67.12}, {250.213}}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest5() { + String sql = + "select min_value(bigNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; + long[] ans = + new long[] {2147483646L, 2147483650L, 2147468648L, 2146483648L, 2107483648L, 3147483648L}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + long actualMinValue = resultSet.getLong(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]], actualMinValue, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest6() { + String sql = + "select min_value(num)+min_value(bigNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; + long[] ans = + new long[] {2147483647L, 2147483654L, 2147468659L, 2146483660L, 2107483661L, 3147483663L}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualMinValue = resultSet.getDouble(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]], actualMinValue, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest7() { + String sql = + "select avg(num)+min_value(floatNum) from root.sg.d group by session(10000ms) order by max_value(floatNum)"; + double[][] ans = + new double[][] { + {13.0, 54.12, 54.12}, + {11.0, 54.121, 54.121}, + {13.0, 231.34, 45.231}, + {15.0, 235.213, 235.213}, + {6.4, 1231.21, 56.32}, + {4.6, 4654.231, 12.123} + }; + long[] times = + new long[] {41536900000L, 31536100000L, 41536000000L, 51536000000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i][0] + ans[i][2], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest8() { + String sql = + "select avg(num)+avg(floatNum) from root.sg.d group by session(10000ms) order by avg(floatNum)+avg(num)"; + double[][] ans = + new double[][] {{1079.56122}, {395.4584}, {65.121}, {151.2855}, {67.12}, {250.213}}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + // 4. raw data query with align by device + private void testNormalOrderByAlignByDevice(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d1"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + if (device.equals("d1")) { + device = "d2"; + } else { + device = "d1"; + i++; + } + total++; + } + assertEquals(i, ans.length); + assertEquals(total, ans.length * 2); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void alignByDeviceOrderByTest1() { + String sql = + "select Time,device,num+bigNum from table0 order by num+floatNum desc, floatNum desc, device asc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + String device = "d1"; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + double actualNum = resultSet.getLong(3); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Long.parseLong(RES[ans[i]][2]), actualNum, 0.0001); + if (device.equals("d1")) { + device = "d2"; + } else { + device = "d1"; + i++; + } + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void alignByDeviceOrderByTest2() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by num, device asc"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest3() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by floatNum, device asc"; + int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest4() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by str, device asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest5() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by num desc, device asc"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + testNormalOrderByAlignByDevice(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void alignByDeviceOrderByTest6() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by str desc, device asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderByAlignByDevice(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void alignByDeviceOrderByTest7() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bool asc, num desc, device asc"; + int[] ans = {12, 11, 10, 6, 3, 8, 7, 0, 14, 13, 4, 9, 5, 1, 2}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest8() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, floatNum desc, device asc"; + int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest9() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by str desc, floatNum desc, device asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderByAlignByDevice(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + private void testNormalOrderByMixAlignBy(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d1"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + if (device.equals("d2")) { + i++; + device = "d1"; + } else { + device = "d2"; + } + + total++; + } + assertEquals(total, ans.length * 2); + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + private void testDeviceViewOrderByMixAlignBy(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d2"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + i++; + total++; + if (total == ans.length) { + i = 0; + if (device.equals("d2")) { + device = "d1"; + } else { + device = "d2"; + } + } + } + assertEquals(total, ans.length * 2); + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + private void orderByBigNumAlignByDevice(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d1"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + if (total < 4) { + i = total % 2; + if (total < 2) { + device = "d2"; + } else { + device = "d1"; + } + } + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + if (device.equals("d2")) { + device = "d1"; + } else { + i++; + device = "d2"; + } + + total++; + } + assertEquals(i, ans.length); + assertEquals(total, ans.length * 2); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void alignByDeviceOrderByTest12() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, device desc, time asc"; + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + orderByBigNumAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest13() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, time desc, device asc"; + int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest14() { + int[] ans = {14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by time desc, bigNum desc, device asc"; + testNormalOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest15() { + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by device desc, bigNum desc, time asc"; + testDeviceViewOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest16() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by device desc, time asc, bigNum desc"; + int[] ans = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; + testDeviceViewOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest17() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, device desc, num asc, time asc"; + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + orderByBigNumAlignByDevice(sql, ans); + } + + // 5. aggregation query align by device + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest() { + String sql = + "select avg(num) from root.** group by session(10000ms) order by avg(num) align by device"; + + double[] ans = {4.6, 4.6, 6.4, 6.4, 11.0, 11.0, 13.0, 13.0, 13.0, 13.0, 15.0, 15.0}; + long[] times = + new long[] { + 0L, + 0L, + 31536000000L, + 31536000000L, + 31536100000L, + 31536100000L, + 41536000000L, + 41536900000L, + 41536000000L, + 41536900000L, + 51536000000L, + 51536000000L + }; + String[] device = + new String[] { + "root.sg.d", + "root.sg.d2", + "root.sg.d", + "root.sg.d2", + "root.sg.d", + "root.sg.d2", + "root.sg.d", + "root.sg.d", + "root.sg.d2", + "root.sg.d2", + "root.sg.d", + "root.sg.d2" + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + double actualAvg = resultSet.getDouble(3); + + assertEquals(device[i], actualDevice); + assertEquals(times[i], actualTime); + assertEquals(ans[i], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + @Ignore + public void orderByInAggregationAlignByDeviceTest2() { + String sql = "select avg(num) from root.** order by avg(num) align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + private void checkSingleDouble(String sql, Object value, boolean deviceAsc) { + String device = "root.sg.d"; + if (!deviceAsc) device = "root.sg.d2"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + String deviceName = resultSet.getString(1); + double actualVal = resultSet.getDouble(2); + assertEquals(deviceName, device); + assertEquals(Double.parseDouble(value.toString()), actualVal, 1); + if (device.equals("root.sg.d")) device = "root.sg.d2"; + else device = "root.sg.d"; + i++; + } + assertEquals(i, 2); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest3() { + String sql = + "select avg(num)+avg(bigNum) from root.** order by max_value(floatNum) align by device"; + long value = 2388936669L + 8; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest4() { + + String sql = + "select avg(num)+avg(bigNum) from root.** order by max_value(floatNum)+min_value(num) align by device"; + long value = 2388936669L + 8; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest5() { + String sql = + "select avg(num) from root.** order by max_value(floatNum)+avg(num) align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest6() { + String sql = + "select avg(num) from root.** order by max_value(floatNum)+avg(num), device asc, time desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest7() { + String sql = + "select avg(num) from root.** order by max_value(floatNum)+avg(num), time asc, device desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, false); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest8() { + String sql = + "select avg(num) from root.** order by time asc, max_value(floatNum)+avg(num), device desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, false); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest9() { + String sql = + "select avg(num) from root.** order by device asc, max_value(floatNum)+avg(num), time desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest10() { + String sql = + "select avg(num) from root.** order by max_value(floatNum) desc,time asc, avg(num) asc, device desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, false); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest11() { + String sql = + "select avg(num) from root.** order by max_value(floatNum) desc,device asc, avg(num) asc, time desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest12() { + String sql = + "select avg(num+floatNum) from root.** order by time,avg(num+floatNum) align by device"; + String value = "537.34154"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest13() { + String sql = "select avg(num) from root.** order by time,avg(num+floatNum) align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest14() { + String sql = "select avg(num+floatNum) from root.** order by time,avg(num) align by device"; + String value = "537.34154"; + checkSingleDouble(sql, value, true); + } + + String[][] UDFRes = + new String[][] { + {"0", "3", "0", "0"}, + {"20", "2", "0", "0"}, + {"40", "1", "0", "0"}, + {"80", "9", "0", "0"}, + {"100", "8", "0", "0"}, + {"31536000000", "6", "0", "0"}, + {"31536000100", "10", "0", "0"}, + {"31536000500", "4", "0", "0"}, + {"31536001000", "5", "0", "0"}, + {"31536010000", "7", "0", "0"}, + {"31536100000", "11", "0", "0"}, + {"41536000000", "12", "2146483648", "0"}, + {"41536000020", "14", "0", "14"}, + {"41536900000", "13", "2107483648", "0"}, + {"51536000000", "15", "0", "15"}, + }; + + // UDF Test + private void orderByUDFTest(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + String time = resultSet.getString(1); + String num = resultSet.getString(2); + String topK = resultSet.getString(3); + String bottomK = resultSet.getString(4); + + assertEquals(time, UDFRes[ans[i]][0]); + assertEquals(num, UDFRes[ans[i]][1]); + if (Objects.equals(UDFRes[ans[i]][3], "0")) { + assertNull(topK); + } else { + assertEquals(topK, UDFRes[ans[i]][3]); + } + + if (Objects.equals(UDFRes[ans[i]][2], "0")) { + assertNull(bottomK); + } else { + assertEquals(bottomK, UDFRes[ans[i]][2]); + } + + i++; + } + assertEquals(i, 15); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByUDFTest1() { + String sql = + "select num, top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2') from root.sg.d order by top_k(num, 'k'='2') nulls first, bottom_k(bigNum, 'k'='2') nulls first"; + int[] ans = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 11, 12, 14}; + orderByUDFTest(sql, ans); + } + + @Ignore + @Test + public void orderByUDFTest2() { + String sql = + "select num, top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2') from root.sg.d order by top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2')"; + int[] ans = {12, 14, 13, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + orderByUDFTest(sql, ans); + } + + private void errorTest(String sql, String error) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.executeQuery(sql); + } catch (Exception e) { + assertEquals(error, e.getMessage()); + } + } + + @Ignore + @Test + public void errorTest1() { + errorTest( + "select num from root.sg.d order by avg(bigNum)", + "701: Raw data and aggregation hybrid query is not supported."); + } + + @Ignore + @Test + public void errorTest2() { + errorTest( + "select avg(num) from root.sg.d order by bigNum", + "701: Raw data and aggregation hybrid query is not supported."); + } + + @Test + public void errorTest3() { + errorTest( + "select bigNum,floatNum from table0 order by s1", + "700: Error occurred while parsing SQL to physical plan: line 1:28 no viable alternative at input 'select bigNum,floatNum from table0'"); + } + + @Ignore + @Test + public void errorTest4() { + errorTest( + "select bigNum,floatNum from root.** order by bigNum", + "701: root.**.bigNum in order by clause shouldn't refer to more than one timeseries."); + } + + @Ignore + @Test + public void errorTest7() { + errorTest( + "select last bigNum,floatNum from root.** order by root.sg.d.bigNum", + "701: root.sg.d.bigNum in order by clause doesn't exist in the result of last query."); + } + + // last query + public void testLastQueryOrderBy(String sql, String[][] ans) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + String time = resultSet.getString(1); + String num = resultSet.getString(2); + String value = resultSet.getString(3); + String dataType = resultSet.getString(4); + + assertEquals(time, ans[0][i]); + assertEquals(num, ans[1][i]); + assertEquals(value, ans[2][i]); + assertEquals(dataType, ans[3][i]); + + i++; + } + assertEquals(i, 4); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void lastQueryOrderBy() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d.num", "root.sg.d2.num", "root.sg.d.bigNum", "root.sg.d2.bigNum"}, + {"15", "15", "3147483648", "3147483648"}, + {"INT32", "INT32", "INT64", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by value, timeseries"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy2() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d2.bigNum", "root.sg.d.num", "root.sg.d.bigNum"}, + {"15", "3147483648", "15", "3147483648"}, + {"INT32", "INT64", "INT32", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by timeseries desc"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy3() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d2.bigNum", "root.sg.d.num", "root.sg.d.bigNum"}, + {"15", "3147483648", "15", "3147483648"}, + {"INT32", "INT64", "INT32", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by timeseries desc, value asc"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy4() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d.num", "root.sg.d2.bigNum", "root.sg.d.bigNum"}, + {"15", "15", "3147483648", "3147483648"}, + {"INT32", "INT32", "INT64", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by value, timeseries desc"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy5() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d.num", "root.sg.d2.bigNum", "root.sg.d.bigNum"}, + {"15", "15", "3147483648", "3147483648"}, + {"INT32", "INT32", "INT64", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by datatype, timeseries desc"; + testLastQueryOrderBy(sql, ans); + } + + private static List TIMES = + Arrays.asList( + 0L, + 20L, + 40L, + 80L, + 100L, + 31536000000L, + 31536000100L, + 31536000500L, + 31536001000L, + 31536010000L, + 31536100000L, + 41536000000L, + 41536000020L, + 41536900000L, + 51536000000L); + + protected static void sessionInsertData1() { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE \"db0\""); + session.executeNonQueryStatement("USE \"db0\""); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("device", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("num", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("bigNum", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("floatNum", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("str", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("bool", TSDataType.BOOLEAN)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + List fieldIds = IMeasurementSchema.getMeasurementNameList(schemaList); + List dataTypes = IMeasurementSchema.getDataTypeList(schemaList); + + List values = + Arrays.asList( + new Object[] {"d1", "a1", 3, 2947483648L, 231.2121, "coconut", false}, + new Object[] {"d1", "a1", 2, 2147483648L, 434.12, "pineapple", true}, + new Object[] {"d1", "a1", 1, 2247483648L, 12.123, "apricot", true}, + new Object[] {"d1", "a1", 9, 2147483646L, 43.12, "apple", false}, + new Object[] {"d1", "a1", 8, 2147483964L, 4654.231, "papaya", true}, + new Object[] {"d1", "a1", 6, 2147483650L, 1231.21, "banana", true}, + new Object[] {"d1", "a1", 10, 3147483648L, 231.55, "pumelo", false}, + new Object[] {"d1", "a1", 4, 2147493648L, 213.1, "peach", false}, + new Object[] {"d1", "a1", 5, 2149783648L, 56.32, "orange", false}, + new Object[] {"d1", "a1", 7, 2147983648L, 213.112, "lemon", true}, + new Object[] {"d1", "a1", 11, 2147468648L, 54.121, "pitaya", false}, + new Object[] {"d1", "a1", 12, 2146483648L, 45.231, "strawberry", false}, + new Object[] {"d1", "a1", 14, 2907483648L, 231.34, "cherry", false}, + new Object[] {"d1", "a1", 13, 2107483648L, 54.12, "lychee", true}, + new Object[] {"d1", "a1", 15, 3147483648L, 235.213, "watermelon", true}); + Tablet tablet = new Tablet("table0", fieldIds, dataTypes, columnTypes, TIMES.size()); + for (int i = 0; i < TIMES.size(); i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, TIMES.get(i)); + for (int j = 0; j < schemaList.size(); j++) { + tablet.addValue(schemaList.get(j).getMeasurementName(), rowIndex, values.get(i)[j]); + } + } + session.insert(tablet); + } catch (Exception e) { + e.printStackTrace(); + } + } + + protected static void sessionInsertData2() { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db0\""); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("device", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("num", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("bigNum", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("floatNum", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("str", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("bool", TSDataType.BOOLEAN)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + List fieldIds = IMeasurementSchema.getMeasurementNameList(schemaList); + List dataTypes = IMeasurementSchema.getDataTypeList(schemaList); + List values = + Arrays.asList( + new Object[] {"d2", "a2", 3, 2947483648L, 231.2121, "coconut", false}, + new Object[] {"d2", "a2", 2, 2147483648L, 434.12, "pineapple", true}, + new Object[] {"d2", "a2", 1, 2247483648L, 12.123, "apricot", true}, + new Object[] {"d2", "a2", 9, 2147483646L, 43.12, "apple", false}, + new Object[] {"d2", "a2", 8, 2147483964L, 4654.231, "papaya", true}, + new Object[] {"d2", "a2", 6, 2147483650L, 1231.21, "banana", true}, + new Object[] {"d2", "a2", 10, 3147483648L, 231.55, "pumelo", false}, + new Object[] {"d2", "a2", 4, 2147493648L, 213.1, "peach", false}, + new Object[] {"d2", "a2", 5, 2149783648L, 56.32, "orange", false}, + new Object[] {"d2", "a2", 7, 2147983648L, 213.112, "lemon", true}, + new Object[] {"d2", "a2", 11, 2147468648L, 54.121, "pitaya", false}, + new Object[] {"d2", "a2", 12, 2146483648L, 45.231, "strawberry", false}, + new Object[] {"d2", "a2", 14, 2907483648L, 231.34, "cherry", false}, + new Object[] {"d2", "a2", 13, 2107483648L, 54.12, "lychee", true}, + new Object[] {"d2", "a2", 15, 3147483648L, 235.213, "watermelon", true}); + Tablet tablet = new Tablet("table0", fieldIds, dataTypes, columnTypes, TIMES.size()); + for (int i = 0; i < TIMES.size(); i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, TIMES.get(i)); + for (int j = 0; j < schemaList.size(); j++) { + tablet.addValue(schemaList.get(j).getMeasurementName(), rowIndex, values.get(i)[j]); + } + } + session.insert(tablet); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBArithmeticTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBArithmeticTableIT.java new file mode 100644 index 0000000000000..a78b87c081e85 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBArithmeticTableIT.java @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBArithmeticTableIT { + + private static final double E = 0.0001; + + private static final String DATABASE_NAME = "test"; + + private static final long MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000; + + private static final double[][] BASE_ANS = {{1, 1, 1.0, 1.0}, {2, 2, 2.0, 2.0}, {3, 3, 3.0, 3.0}}; + + private static final Map> OPERATIONS = new HashMap<>(); + + static { + OPERATIONS.put(" + ", (a, b) -> a + b); + OPERATIONS.put(" - ", (a, b) -> a - b); + OPERATIONS.put(" * ", (a, b) -> a * b); + OPERATIONS.put(" / ", (a, b) -> a / b); + OPERATIONS.put(" % ", (a, b) -> a % b); + } + + private static final String[] INSERTION_SQLS = { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1 (device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 DATE FIELD, s6 TIMESTAMP FIELD, s7 BOOLEAN FIELD, s8 TEXT FIELD)", + "insert into table1(device, time, s1, s2, s3, s4, s5, s6, s7, s8) values ('d1', 1, 1, 1, 1.0, 1.0, '2024-01-01', 10, true, 'test')", + "insert into table1(device, time, s1, s2, s3, s4, s5, s6, s7, s8) values ('d1', 2, 2, 2, 2.0, 2.0, '2024-02-01', 20, true, 'test')", + "insert into table1(device, time, s1, s2, s3, s4, s5, s6, s7, s8) values ('d1', 3, 3, 3, 3.0, 3.0, '2024-03-01', 30, true, 'test')", + "CREATE TABLE table2 (device STRING TAG, date DATE FIELD)", + "insert into table2(device, time, date) values ('d1', 1, '9999-12-31')", + "insert into table2(device, time, date) values ('d1', 2, '1000-01-01')", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(INSERTION_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + public static double[][] calculateExpectedAns(String operator) { + double[][] expectedAns = new double[BASE_ANS.length][BASE_ANS[0].length * BASE_ANS[0].length]; + BiFunction operation = OPERATIONS.get(operator); + if (operation == null) { + throw new IllegalArgumentException("Unsupported operator: " + operator); + } + + for (int i = 0; i < BASE_ANS.length; i++) { + int baseLength = BASE_ANS[i].length; + for (int j = 0; j < baseLength; j++) { + for (int k = 0; k < baseLength; k++) { + expectedAns[i][j * baseLength + k] = operation.apply(BASE_ANS[i][j], BASE_ANS[i][k]); + } + } + } + return expectedAns; + } + + @Test + public void testArithmeticBinaryWithoutDateAndTimestamp() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + + // generate sql + + for (String operator : new String[] {" + ", " - ", " * ", " / ", " % "}) { + List expressions = new ArrayList<>(); + String[] operands = new String[] {"s1", "s2", "s3", "s4"}; + for (String leftOperand : operands) { + for (String rightOperand : operands) { + expressions.add(leftOperand + operator + rightOperand); + } + } + + String sql = String.format("select %s from table1", String.join(",", expressions)); + ResultSet resultSet = statement.executeQuery(sql); + + // generate answer + double[][] expectedAns = calculateExpectedAns(operator); + + // Make sure the number of columns in the result set is correct + assertEquals(expressions.size(), resultSet.getMetaData().getColumnCount()); + + // check the result + for (double[] expectedAn : expectedAns) { + resultSet.next(); + for (int i = 0; i < expectedAn.length; i++) { + double result = Double.parseDouble(resultSet.getString(i + 1)); + assertEquals(expectedAn[i], result, E); + } + } + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testArithmeticBinaryWithDateAndTimestamp() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + + String sql = + "select s1+s5,s1+s6,s2+s5,s2+s6,s5+s1,s5+s2,s6+s1,s6+s2,s5-s1,s5-s2,s6-s1,s6-s2 from table1"; + ResultSet resultSet = statement.executeQuery(sql); + + assertEquals(12, resultSet.getMetaData().getColumnCount()); + + int[][] expectedAns = { + {20240101, 11, 20240101, 11, 20240101, 20240101, 11, 11, 20231231, 20231231, 9, 9}, + {20240201, 22, 20240201, 22, 20240201, 20240201, 22, 22, 20240131, 20240131, 18, 18}, + {20240301, 33, 20240301, 33, 20240301, 20240301, 33, 33, 20240229, 20240229, 27, 27} + }; + + for (int[] expectedAn : expectedAns) { + resultSet.next(); + + for (int i = 0; i < expectedAn.length; i++) { + int result = resultSet.getInt(i + 1); + assertEquals(expectedAn[i], result); + } + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testArithmeticUnary() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + + String[] expressions = new String[] {"- s1", "- s2", "- s3", "- s4"}; + String sql = String.format("select %s from table1", String.join(",", expressions)); + ResultSet resultSet = statement.executeQuery(sql); + + assertEquals(expressions.length, resultSet.getMetaData().getColumnCount()); + + for (int i = 1; i < 4; ++i) { + resultSet.next(); + for (int j = 0; j < expressions.length; ++j) { + double expected = -i; + double actual = Double.parseDouble(resultSet.getString(j + 1)); + assertEquals(expected, actual, E); + } + } + resultSet.close(); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testTimestampNegation() { + String sql = "select -s6 from table1"; + tableResultSetEqualTest( + sql, + new String[] {"_col0"}, + new String[] { + "1969-12-31T23:59:59.990Z,", "1969-12-31T23:59:59.980Z,", "1969-12-31T23:59:59.970Z," + }, + DATABASE_NAME); + } + + @Test + public void testBinaryWrongType() { + + testBinaryDifferentCombinationsFail( + new String[] {" * ", " / ", " % "}, + new String[] {"s1", "s2", "s3", "s4"}, + new String[] {"s5", "s6"}, + "table1", + "701: Cannot apply operator", + "test"); + + testBinaryDifferentCombinationsFail( + new String[] {" * ", " / ", " % "}, + new String[] {"s5", "s6"}, + new String[] {"s1", "s2", "s3", "s4"}, + "table1", + "701: Cannot apply operator", + "test"); + } + + @Test + public void testUnaryWrongType() { + tableAssertTestFail("select -s5 from table1", "701: Cannot negate", "test"); + tableAssertTestFail("select -s7 from table1", "701: Cannot negate", "test"); + tableAssertTestFail("select -s8 from table1", "701: Cannot negate", "test"); + } + + @Test + public void testOverflow() { + // int addition overflow + tableAssertTestFail( + String.format("select s1+%d from table1", Integer.MAX_VALUE), + "750: int Addition overflow", + "test"); + + // int subtraction overflow + tableAssertTestFail( + String.format("select s1-(%d) from table1", Integer.MIN_VALUE), + "750: int Subtraction overflow", + "test"); + + // int multiplication overflow + tableAssertTestFail( + String.format("select (s1+1)*%d from table1", Integer.MAX_VALUE), + "750: int Multiplication overflow", + "test"); + + // Date addition overflow + tableAssertTestFail( + String.format("select s5+%d from table1", Long.MAX_VALUE), + "750: long Addition overflow", + "test"); + + // Date subtraction overflow + tableAssertTestFail( + String.format("select s5-(%d) from table1", Long.MIN_VALUE), + "750: long Subtraction overflow", + "test"); + + // long addition overflow + tableAssertTestFail( + String.format("select s2+%d from table1", Long.MAX_VALUE), + "750: long Addition overflow", + "test"); + + // long subtraction overflow + tableAssertTestFail( + String.format("select s2-(%d) from table1", Long.MIN_VALUE), + "750: long Subtraction overflow", + "test"); + + // Timestamp addition overflow + tableAssertTestFail( + String.format("select s6+%d from table1", Long.MAX_VALUE), + "750: long Addition overflow", + "test"); + + // Timestamp subtraction overflow + tableAssertTestFail( + String.format("select s6-(%d) from table1", Long.MIN_VALUE), + "750: long Subtraction overflow", + "test"); + } + + @Test + public void testFloatDivisionByZeroSpecialCase() { + String[] expectedHeader = new String[5]; + for (int i = 0; i < expectedHeader.length; i++) { + expectedHeader[i] = "_col" + i; + } + String[] expectedAns = { + "Infinity,0.0,NaN,-0.0,-Infinity,", + "Infinity,0.0,NaN,-0.0,-Infinity,", + "Infinity,0.0,NaN,-0.0,-Infinity," + }; + + tableResultSetEqualTest( + "select s3/0.0,0.0/s3,0.0/0.0,0.0/-s3,-s3/0.0 from table1", + expectedHeader, + expectedAns, + "test"); + } + + @Test + public void testDoubleDivisionByZeroSpecialCase() { + String[] expectedHeader = new String[5]; + for (int i = 0; i < expectedHeader.length; i++) { + expectedHeader[i] = "_col" + i; + } + String[] expectedAns = {"NaN,0.0,NaN,0.0,NaN,", "NaN,0.0,NaN,0.0,NaN,", "NaN,0.0,NaN,0.0,NaN,"}; + + tableResultSetEqualTest( + "select s3%0.0,0.0%s3,0.0%0.0,0.0%-s3,-s3%0.0 from table1", + expectedHeader, expectedAns, "test"); + } + + @Test + public void testDivisionByZero() { + tableAssertTestFail("select s1/0 from table1", "751: Division by zero", "test"); + tableAssertTestFail("select s2/0 from table1", "751: Division by zero", "test"); + + tableAssertTestFail("select s1%0 from table1", "751: Division by zero", "test"); + tableAssertTestFail("select s2%0 from table1", "751: Division by zero", "test"); + } + + @Test + public void testDateOutOfRange() { + tableAssertTestFail( + String.format( + "select date + %s from table2 where time = 1", + DateTimeUtils.correctPrecision(MILLISECONDS_IN_DAY)), + "752: Year must be between 1000 and 9999.", + "test"); + + tableAssertTestFail( + String.format( + "select %s + date from table2 where time = 1", + DateTimeUtils.correctPrecision(MILLISECONDS_IN_DAY)), + "752: Year must be between 1000 and 9999.", + "test"); + + tableAssertTestFail( + String.format("select %s + date from table2 where time = 1", 86400000), + "752: Year must be between 1000 and 9999.", + "test"); + + tableAssertTestFail( + String.format( + "select date - %s from table2 where time = 2", + DateTimeUtils.correctPrecision(MILLISECONDS_IN_DAY)), + "752: Year must be between 1000 and 9999.", + "test"); + } + + private void testBinaryDifferentCombinationsFail( + String[] operators, + String[] leftOperands, + String[] rightOperands, + String tableName, + String errMsg, + String databaseName) { + for (String operator : operators) { + for (String leftOperand : leftOperands) { + for (String rightOperand : rightOperands) { + tableAssertTestFail( + String.format( + "select %s %s %s from %s", leftOperand, operator, rightOperand, tableName), + errMsg, + databaseName); + } + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBFuzzyQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBFuzzyQueryTableIT.java new file mode 100644 index 0000000000000..4a914c1f98508 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBFuzzyQueryTableIT.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFuzzyQueryTableIT { + private static List sqls = new ArrayList<>(); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + initCreateSQLStatement(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void initCreateSQLStatement() { + sqls.add("CREATE DATABASE TestFuzzyQuery"); + sqls.add("USE TestFuzzyQuery"); + sqls.add( + "CREATE TABLE likeTest(device_id STRING TAG, s1 TEXT FIELD, s2 STRING FIELD, s3 STRING FIELD)"); + sqls.add( + "INSERT INTO likeTest(time, device_id, s1, s2, s3) VALUES(1,'d1', 'abcdef', 'a%', '\\')"); + sqls.add( + "INSERT INTO likeTest(time, device_id, s1, s2, s3) VALUES(2,'d1', '_abcdef', '\\_a%','\\')"); + sqls.add( + "INSERT INTO likeTest(time, device_id, s1, s2, s3) VALUES(3,'d1', 'abcdef%', '%_\\%','\\')"); + } + + private static void insertData() throws SQLException { + Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement(); + + for (String sql : sqls) { + statement.execute(sql); + } + statement.close(); + } + + @Test + public void testLike() throws SQLException { + Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement(); + statement.execute("USE TestFuzzyQuery"); + String[] ans = new String[] {"abcdef"}; + String query = "SELECT s1 FROM likeTest where s1 LIKE s2"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 0; i < 1; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"abcdef", "_abcdef", "abcdef%"}; + query = "SELECT s1 FROM likeTest where s1 LIKE s2 ESCAPE s3"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 0; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java new file mode 100644 index 0000000000000..32e5067d8c605 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java @@ -0,0 +1,307 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBGreatestLeastTableIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE boolean_table(device_id STRING TAG, bool1 BOOLEAN FIELD, bool2 BOOLEAN FIELD)", + "CREATE TABLE number_table(device_id STRING TAG, int1 INT32 FIELD, int2 INT32 FIELD, long1 INT64 FIELD, long2 INT64 FIELD, float1 FLOAT FIELD, float2 FLOAT FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD)", + "CREATE TABLE string_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, text1 TEXT FIELD, text2 TEXT FIELD)", + "CREATE TABLE mix_type_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 STRING FIELD, s7 TEXT FIELD)", + "CREATE TABLE null_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, int1 INT32 FIELD, int2 INT32 FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD, timestamp1 TIMESTAMP FIELD, timestamp2 TIMESTAMP FIELD)", + "CREATE TABLE any_null_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, int1 INT32 FIELD, int2 INT32 FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD, timestamp1 TIMESTAMP FIELD, timestamp2 TIMESTAMP FIELD)", + // normal case + "INSERT INTO number_table(time, device_id, int1, int2, long1, long2, float1, float2, double1, double2) VALUES (10, 'd1', 1000000, 2000000, 1000000, 2000000, 10.1, 20.2, 10.1, 20.2)", + "INSERT INTO string_table(time, device_id, string1, string2, text1, text2) VALUES(10, 'd1', 'aaa', 'bbb', 'aaa', 'bbb')", + "INSERT INTO boolean_table(time, device_id, bool1, bool2) VALUES(10, 'd1', true, false)", + "INSERT INTO boolean_table(time, device_id, bool1, bool2) VALUES(20, 'd1', false, true)", + "INSERT INTO boolean_table(time, device_id, bool1, bool2) VALUES(30, 'd1', true, true)", + "INSERT INTO mix_type_table(time, device_id, s1, s2, s3, s4, s5, s6, s7) VALUES(10, 'd1', 1, 1, 1.0, 1.0, true, 'a', 'a')", + "INSERT INTO null_table(time, device_id, string1, string2) VALUES(10, 'd1', null, null)", + "INSERT INTO any_null_table(time, device_id, string2, int2, double2, timestamp2) VALUES(10, 'd1', 'test', 10, 10.0, 10)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testNumberTypeGreatestFunction() { + + tableResultSetEqualTest( + "SELECT GREATEST(int1, int2) FROM number_table", + new String[] {"_col0"}, + new String[] {"2000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(long1, long2) FROM number_table", + new String[] {"_col0"}, + new String[] {"2000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(float1, float2) FROM number_table", + new String[] {"_col0"}, + new String[] {"20.2,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(double1, double2) FROM number_table", + new String[] {"_col0"}, + new String[] {"20.2,"}, + DATABASE_NAME); + } + + @Test + public void testNumberTypeLeastFunction() { + tableResultSetEqualTest( + "SELECT LEAST(int1, int2) FROM number_table", + new String[] {"_col0"}, + new String[] {"1000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(long1, long2) FROM number_table", + new String[] {"_col0"}, + new String[] {"1000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(float1, float2) FROM number_table", + new String[] {"_col0"}, + new String[] {"10.1,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(double1, double2) FROM number_table", + new String[] {"_col0"}, + new String[] {"10.1,"}, + DATABASE_NAME); + } + + @Test + public void testStringTypeGreatestFunction() { + tableResultSetEqualTest( + "SELECT GREATEST(string1, string2) FROM string_table", + new String[] {"_col0"}, + new String[] {"bbb,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(text1, text2) FROM string_table", + new String[] {"_col0"}, + new String[] {"bbb,"}, + DATABASE_NAME); + } + + @Test + public void testStringTypeLeastFunction() { + tableResultSetEqualTest( + "SELECT LEAST(string1, string2) FROM string_table", + new String[] {"_col0"}, + new String[] {"aaa,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(text1, text2) FROM string_table", + new String[] {"_col0"}, + new String[] {"aaa,"}, + DATABASE_NAME); + } + + @Test + public void testBooleanTypeGreatestFunction() { + tableResultSetEqualTest( + "SELECT GREATEST(bool1, bool2) FROM boolean_table", + new String[] {"_col0"}, + new String[] {"true,", "true,", "true,"}, + DATABASE_NAME); + } + + @Test + public void testBooleanTypeLeastFunction() { + tableResultSetEqualTest( + "SELECT LEAST(bool1, bool2) FROM boolean_table", + new String[] {"_col0"}, + new String[] {"false,", "false,", "true,"}, + DATABASE_NAME); + } + + @Test + public void testAllNullValue() { + tableResultSetEqualTest( + "SELECT GREATEST(string1, string2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(string1, string2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(int1, int2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(int1, int2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(double1, double2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(double1, double2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(timestamp1, timestamp2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(timestamp1, timestamp2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + } + + @Test + public void testAnyNullValue() { + tableResultSetEqualTest( + "SELECT GREATEST(string1, string2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"test,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(string1, string2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"test,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(int1, int2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(int1, int2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(double1, double2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(double1, double2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(timestamp1, timestamp2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"1970-01-01T00:00:00.010Z,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(timestamp1, timestamp2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"1970-01-01T00:00:00.010Z,"}, + DATABASE_NAME); + } + + @Test + public void testAnomalies() { + // do not support different type + for (int i = 1; i <= 7; i++) { + for (int j = i + 1; j <= 7; j++) { + tableAssertTestFail( + String.format("SELECT LEAST(s%d, s%d) FROM mix_type_table", i, j), + "701: Scalar function least must have at least two arguments, and all type must be the same.", + DATABASE_NAME); + + tableAssertTestFail( + String.format("SELECT GREATEST(s%d, s%d) FROM mix_type_table", i, j), + "701: Scalar function greatest must have at least two arguments, and all type must be the same.", + DATABASE_NAME); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBInTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBInTableIT.java new file mode 100644 index 0000000000000..c8ffc4c0f47cd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBInTableIT.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBInTableIT { + private static final String DATABASE_NAME = "test"; + private static String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE sg(device1 STRING TAG, device2 STRING TAG, qrcode TEXT FIELD, date_v DATE FIELD, blob_v BLOB FIELD, timestamp_v TIMESTAMP FIELD)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465600000,'d1','s1','qrcode001', '2024-08-01', X'abc0',1509465600000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465660000,'d1','s1','qrcode002', '2024-08-02', X'abc1',1509465660000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465720000,'d1','s1','qrcode003', '2024-08-03', X'abc2',1509465720000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465780000,'d1','s1','qrcode004', '2024-08-04', X'abc3',1509465780000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465720000,'d1','s2','qrcode002', '2024-08-05', X'abc4',1509465720000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465780000,'d1','s2','qrcode003', '2024-08-06', X'abc5',1509465780000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465840000,'d1','s2','qrcode004', '2024-08-07', X'abc6',1509465840000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465900000,'d1','s2','qrcode005', '2024-08-08', X'abc7',1509465900000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465780000,'d2','s1','qrcode002', '2024-08-09', X'abc8',1509465780000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465840000,'d2','s1','qrcode003', '2024-08-10', X'abc9',1509465840000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465900000,'d2','s1','qrcode004', '2024-08-11', X'abca',1509465900000)", + "insert into sg(time,device1,device2,qrcode,date_v,blob_v,timestamp_v) values(1509465960000,'d2','s1','qrcode005', '2024-08-12', X'abcb',1509465960000)", + "CREATE TABLE table1(device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + + prepareTableData(sqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCastException() { + tableAssertTestFail( + "select * from table1 where s1 in ('test')", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": IN value and list items must be the same type or coercible to a common type.", + DATABASE_NAME); + tableAssertTestFail( + "select * from table1 where s2 in ('test')", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": IN value and list items must be the same type or coercible to a common type.", + DATABASE_NAME); + tableAssertTestFail( + "select * from table1 where s3 in ('test')", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": IN value and list items must be the same type or coercible to a common type.", + DATABASE_NAME); + tableAssertTestFail( + "select * from table1 where s4 in ('test')", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": IN value and list items must be the same type or coercible to a common type.", + DATABASE_NAME); + tableAssertTestFail( + "select * from table1 where s5 in ('test')", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": IN value and list items must be the same type or coercible to a common type.", + DATABASE_NAME); + } + + @Test + public void selectWithAlignByDeviceTest() { + String[] retArray = + new String[] { + defaultFormatDataTime(1509465660000L) + ",d1,s1,qrcode002,", + defaultFormatDataTime(1509465780000L) + ",d1,s1,qrcode004,", + defaultFormatDataTime(1509465720000L) + ",d1,s2,qrcode002,", + defaultFormatDataTime(1509465840000L) + ",d1,s2,qrcode004,", + defaultFormatDataTime(1509465780000L) + ",d2,s1,qrcode002,", + defaultFormatDataTime(1509465900000L) + ",d2,s1,qrcode004,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery( + "select time,device1,device2,qrcode from sg where qrcode in ('d1','s1','qrcode002','qrcode004') order by device1,device2")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device1,device2,qrcode,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(6, cnt); + } + + // DATE IN + try (ResultSet resultSet = + statement.executeQuery( + "select date_v from sg where date_v in (CAST('2024-08-05' AS DATE), CAST('2024-08-12' AS DATE), CAST('2024-08-22' AS DATE)) order by date_v")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader( + resultSetMetaData, + "date_v,", + new int[] { + Types.DATE, + }); + + int cnt = 0; + String expectedString = "2024-08-05,2024-08-12,"; + StringBuilder actualBuilder = new StringBuilder(); + while (resultSet.next()) { + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + } + cnt++; + } + Assert.assertEquals(expectedString, actualBuilder.toString()); + Assert.assertEquals(2, cnt); + } + + // BLOB IN + try (ResultSet resultSet = + statement.executeQuery( + "select blob_v from sg where blob_v in (X'abc3', X'abca', X'bbbb') order by blob_v")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + checkHeader( + resultSetMetaData, + "blob_v,", + new int[] { + Types.BLOB, + }); + + String expectedString = "0xabc3,0xabca,"; + StringBuilder actualBuilder = new StringBuilder(); + int cnt = 0; + while (resultSet.next()) { + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + } + cnt++; + } + Assert.assertEquals(expectedString, actualBuilder.toString()); + Assert.assertEquals(2, cnt); + } + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testTimestampIn() { + // select time,device1,device2,timestamp_v from sg where timestamp_v in (1509465600000) + // 1509465600000,'d1','s1', + String[] expectedHeader = new String[] {"time", "device1", "device2", "timestamp_v"}; + String[] retArray = + new String[] { + "2017-10-31T16:00:00.000Z,d1,s1,2017-10-31T16:00:00.000Z,", + }; + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v in (1509465600000)", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v=1509465600000", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v=2017-11-01T00:00:00.000+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v=CAST('2017-11-01T00:00:00.000+08:00' AS TIMESTAMP)", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v in (1509465600000.0)", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v=1509465600000.0", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2017-10-31T16:03:00.000Z,d1,s1,2017-10-31T16:03:00.000Z,", + "2017-10-31T16:03:00.000Z,d1,s2,2017-10-31T16:03:00.000Z,", + "2017-10-31T16:04:00.000Z,d1,s2,2017-10-31T16:04:00.000Z,", + "2017-10-31T16:05:00.000Z,d1,s2,2017-10-31T16:05:00.000Z,", + "2017-10-31T16:03:00.000Z,d2,s1,2017-10-31T16:03:00.000Z,", + "2017-10-31T16:04:00.000Z,d2,s1,2017-10-31T16:04:00.000Z,", + "2017-10-31T16:05:00.000Z,d2,s1,2017-10-31T16:05:00.000Z,", + "2017-10-31T16:06:00.000Z,d2,s1,2017-10-31T16:06:00.000Z,", + }; + tableResultSetEqualTest( + "select time,device1,device2,timestamp_v from sg where timestamp_v not in (2017-11-01T00:00:00.000+08:00,1509465660000.0,CAST('2017-11-01T00:02:00.000+08:00' AS TIMESTAMP)) order by device1,device2,time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java new file mode 100644 index 0000000000000..27d14c0e1fccb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNoSelectExpressionAfterAnalyzedTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE sg(device STRING TAG, s1 INT32 FIELD)", + "insert into sg(time,device,s1) values(1,'d1',1)", + "insert into sg(time,device,s1,s2) values(1,'d1',1,1)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAlignByDevice() { + String[] expectedHeader = new String[] {TIMESTAMP_STR}; + String[] retArray = new String[] {}; + tableAssertTestFail( + "select s2 from sg where s1>0 order by device", + "616: Column 's2' cannot be resolved", + DATABASE_NAME); + + // TODO After Aggregation supported + /*tableResultSetEqualTest( + "select count(s2) from sg where s1>0 order by device", expectedHeader, retArray,DATABASE_NAME);*/ + + // mix test + /* expectedHeader = new String[] {DEVICE, count(s1), count(s2)}; + retArray = new String[] {"sg,1,null,", "root.sg.d2,1,1,"}; + tableResultSetEqualTest( + "select count(s1), count(s2) from sg where s1>0 order by device", + expectedHeader, + retArray,DATABASE_NAME);*/ + + tableAssertTestFail( + "select s1, s2 from sg where s1>0 order by device", + "616: Column 's2' cannot be resolved", + DATABASE_NAME); + } + + @Test + public void testAlignByTime() { + tableAssertTestFail( + "select s2 from sg where s1>0", "616: Column 's2' cannot be resolved", DATABASE_NAME); + + /*tableResultSetEqualTest("select count(s2) from sg where s1>0", expectedHeader, retArray,DATABASE_NAME);*/ + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullOperandTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullOperandTableIT.java new file mode 100644 index 0000000000000..eef6773f03f08 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullOperandTableIT.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullOperandTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE sg1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 BOOLEAN FIELD, s4 BOOLEAN FIELD, s5 TEXT FIELD)", + "INSERT INTO sg1(time,device,s1,s3,s4) values(1, 'd1', 1, true, false)", + "INSERT INTO sg1(time,device,s1,s3) values(2, 'd1', 2, true)", + "INSERT INTO sg1(time,device,s1,s4) values(3, 'd1', 3, false)", + "flush", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /** + * ArithmeticOperations: +, -, *, /, % + * + *

The result will be NULL if any Operand of ArithmeticOperations is NULL. + */ + @Test + public void testArithmeticOperations() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", + }; + String[] retArray = + new String[] { + "null,null,null,null,null,null,", + "null,null,null,null,null,null,", + "null,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select s1+s2, s1-s2, s1*s2, s1/s2, s1%s2, s2%s2 from sg1", + expectedHeader, retArray, DATABASE_NAME); + } + + /** + * CompareOperations: =, >, <, like, in, between...(don't include `is null`) + * + *

The result will be NULL if any Operand of CompareOperations is NULL. + */ + @Test + public void testCompareOperations() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", + }; + String[] retArray = + new String[] { + "null,null,null,null,null,null,", + "null,null,null,null,null,null,", + "null,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select s1=s2, s1>s2, s1See + * 'https://learn.microsoft.com/zh-cn/sql/connect/ado-net/sql/handle-null-values?view=sql-server-ver16' + */ + @Test + public void testLogicOperations() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", + }; + String[] retArray = + new String[] { + "true,false,true,false,", "true,null,true,null,", "null,false,null,false,", + }; + tableResultSetEqualTest( + "select s3 or s3, s4 and s4, s3 or s4, s3 and s4 from sg1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testWhere() { + String[] expectedHeader = + new String[] { + "s1", "s3", "s4", + }; + String[] retArray = new String[] {}; + tableResultSetEqualTest( + "select s1, s3, s4 from sg1 where s2>0", expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select s1, s3, s4 from sg1 where s2 is not null", expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "1,true,false,", "2,true,null,", "3,null,false,", + }; + tableResultSetEqualTest( + "select s1, s3, s4 from sg1 where s2 is null and s5 is null", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1,true,false,", "2,true,null,", "3,null,false,", + }; + expectedHeader = + new String[] { + "s1", "s3", "s4", + }; + tableResultSetEqualTest( + "select s1, s3, s4 from sg1 where s2 is null and s5 is null order by device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Ignore // TODO + @Test + public void testHaving() { + String[] expectedHeader = + new String[] { + TIMESTAMP_STR, "count(s1)", "count(s2)", + }; + String[] retArray = new String[] {}; + tableResultSetEqualTest( + "select count(s1), count(s2) from root.** group by ([1,4),1ms) having count(s2)>0", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1,1,0,", "2,1,0,", "3,1,0,", + }; + tableResultSetEqualTest( + "select count(s1), count(s2) from root.** group by ([1,4),1ms) having count(s2)>=0", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1,root.test.sg1,1,0,", "2,root.test.sg1,1,0,", "3,root.test.sg1,1,0,", + }; + expectedHeader = + new String[] { + TIMESTAMP_STR, "Device", "count(s1)", "count(s2)", + }; + tableResultSetEqualTest( + "select count(s1), count(s2) from root.** group by ([1,4),1ms) having count(s2)>=0 align by device", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBPaginationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBPaginationTableIT.java new file mode 100644 index 0000000000000..8aece16bd7db7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBPaginationTableIT.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPaginationTableIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE vehicle(device STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD)", + "insert into vehicle(time,device,s0) values(1,'d1',101)", + "insert into vehicle(time,device,s0) values(2,'d1',198)", + "insert into vehicle(time,device,s0) values(100,'d1',99)", + "insert into vehicle(time,device,s0) values(101,'d1',99)", + "insert into vehicle(time,device,s0) values(102,'d1',80)", + "insert into vehicle(time,device,s0) values(103,'d1',99)", + "insert into vehicle(time,device,s0) values(104,'d1',90)", + "insert into vehicle(time,device,s0) values(105,'d1',99)", + "insert into vehicle(time,device,s0) values(106,'d1',99)", + "insert into vehicle(time,device,s0) values(2,'d1',10000)", + "insert into vehicle(time,device,s0) values(50,'d1',10000)", + "insert into vehicle(time,device,s0) values(1000,'d1',22222)", + "insert into vehicle(time,device,s1) values(1,'d1',1101)", + "insert into vehicle(time,device,s1) values(2,'d1',198)", + "insert into vehicle(time,device,s1) values(100,'d1',199)", + "insert into vehicle(time,device,s1) values(101,'d1',199)", + "insert into vehicle(time,device,s1) values(102,'d1',180)", + "insert into vehicle(time,device,s1) values(103,'d1',199)", + "insert into vehicle(time,device,s1) values(104,'d1',190)", + "insert into vehicle(time,device,s1) values(105,'d1',199)", + "insert into vehicle(time,device,s1) values(2,'d1',40000)", + "insert into vehicle(time,device,s1) values(50,'d1',50000)", + "insert into vehicle(time,device,s1) values(1000,'d1',55555)", + "insert into vehicle(time,device,s2) values(1000,'d1',55555)", + "insert into vehicle(time,device,s2) values(2,'d1',2.22)", + "insert into vehicle(time,device,s2) values(3,'d1',3.33)", + "insert into vehicle(time,device,s2) values(4,'d1',4.44)", + "insert into vehicle(time,device,s2) values(102,'d1',10.00)", + "insert into vehicle(time,device,s2) values(105,'d1',11.11)", + "insert into vehicle(time,device,s2) values(1000,'d1',1000.11)", + "insert into vehicle(time,device,s1) values(2000-01-01T08:00:00+08:00, 'd1',100)", + "CREATE TABLE db(device STRING TAG, s1 INT32 FIELD)", + "insert into db(time,device,s1) values(0,'d1', 0)", + "insert into db(time,device,s1) values(1,'d1', 1)", + "insert into db(time,device,s1) values(2,'d1', 2)", + "insert into db(time,device,s1) values(3,'d1', 3)", + "insert into db(time,device,s1) values(4,'d1', 4)", + "insert into db(time,device,s1) values(5,'d1', 5)", + "insert into db(time,device,s1) values(6,'d1', 6)", + "insert into db(time,device,s1) values(7,'d1', 7)", + "insert into db(time,device,s1) values(8,'d1', 8)", + "insert into db(time,device,s1) values(9,'d1', 9)", + "insert into db(time,device,s1) values(10,'d1', 10)", + "insert into db(time,device,s1) values(11,'d1', 11)", + "insert into db(time,device,s1) values(12,'d1', 12)", + "insert into db(time,device,s1) values(13,'d1', 13)", + "insert into db(time,device,s1) values(14,'d1', 14)" + }; + + @BeforeClass + public static void setUp() throws InterruptedException { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void rawDataQueryTest() { + List querySQLs = + Arrays.asList( + "SELECT time,s1 FROM vehicle WHERE time<200 limit 3", + "SELECT time,s0 FROM vehicle WHERE s1 > 190 limit 3", + "SELECT time,s1,s2 FROM vehicle WHERE s1 > 190 or s2 < 10.0 offset 2 limit 3", + "SELECT time,s2 FROM vehicle WHERE s1 > 190 or s2 < 10.0 offset 1 limit 3"); + List> expectHeaders = + Arrays.asList( + Arrays.asList("time", "s1"), + Arrays.asList("time", "s0"), + Arrays.asList("time", "s1", "s2"), + Arrays.asList("time", "s2")); + List retArrays = + Arrays.asList( + new String[] { + defaultFormatDataTime(1) + ",1101,", + defaultFormatDataTime(2) + ",40000,", + defaultFormatDataTime(3) + ",null,", + }, + new String[] { + defaultFormatDataTime(1) + ",101,", + defaultFormatDataTime(2) + ",10000,", + defaultFormatDataTime(50) + ",10000," + }, + new String[] { + defaultFormatDataTime(3) + ",null,3.33,", + defaultFormatDataTime(4) + ",null,4.44,", + defaultFormatDataTime(50) + ",50000,null," + }, + new String[] { + defaultFormatDataTime(2) + ",2.22,", + defaultFormatDataTime(3) + ",3.33,", + defaultFormatDataTime(4) + ",4.44,", + }); + + for (int i = 0; i < querySQLs.size(); i++) { + tableResultSetEqualTest( + querySQLs.get(i), + expectHeaders.get(i).toArray(new String[0]), + retArrays.get(i), + DATABASE_NAME); + } + } + + @Test + public void limitOffsetPushDownTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = + new String[] { + "3,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 1 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 3 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "7,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 5 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "12,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 10 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBQueryDemoTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBQueryDemoTableIT.java new file mode 100644 index 0000000000000..8323558b68a94 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBQueryDemoTableIT.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBQueryDemoTableIT { + private static final String DATABASE_NAME = "test"; + private static String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE wf(device STRING TAG, status BOOLEAN FIELD, temperature FLOAT FIELD, hardware TEXT FIELD)", + "insert into wf(time,device,status) values(1509465600000,'wt01',true)", + "insert into wf(time,device,status) values(1509465660000,'wt01',true)", + "insert into wf(time,device,status) values(1509465720000,'wt01',false)", + "insert into wf(time,device,status) values(1509465780000,'wt01',false)", + "insert into wf(time,device,status) values(1509465840000,'wt01',false)", + "insert into wf(time,device,status) values(1509465900000,'wt01',false)", + "insert into wf(time,device,status) values(1509465960000,'wt01',false)", + "insert into wf(time,device,status) values(1509466020000,'wt01',false)", + "insert into wf(time,device,status) values(1509466080000,'wt01',false)", + "insert into wf(time,device,status) values(1509466140000,'wt01',false)", + "insert into wf(time,device,temperature) values(1509465600000,'wt01',25.957603)", + "insert into wf(time,device,temperature) values(1509465660000,'wt01',24.359503)", + "insert into wf(time,device,temperature) values(1509465720000,'wt01',20.092794)", + "insert into wf(time,device,temperature) values(1509465780000,'wt01',20.182663)", + "insert into wf(time,device,temperature) values(1509465840000,'wt01',21.125198)", + "insert into wf(time,device,temperature) values(1509465900000,'wt01',22.720892)", + "insert into wf(time,device,temperature) values(1509465960000,'wt01',20.71)", + "insert into wf(time,device,temperature) values(1509466020000,'wt01',21.451046)", + "insert into wf(time,device,temperature) values(1509466080000,'wt01',22.57987)", + "insert into wf(time,device,temperature) values(1509466140000,'wt01',20.98177)", + "insert into wf(time,device,hardware) values(1509465600000,'wt02','v2')", + "insert into wf(time,device,hardware) values(1509465660000,'wt02','v2')", + "insert into wf(time,device,hardware) values(1509465720000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509465780000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509465840000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509465900000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509465960000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509466020000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509466080000,'wt02','v1')", + "insert into wf(time,device,hardware) values(1509466140000,'wt02','v1')", + "insert into wf(time,device,status) values(1509465600000,'wt02',true)", + "insert into wf(time,device,status) values(1509465660000,'wt02',true)", + "insert into wf(time,device,status) values(1509465720000,'wt02',false)", + "insert into wf(time,device,status) values(1509465780000,'wt02',false)", + "insert into wf(time,device,status) values(1509465840000,'wt02',false)", + "insert into wf(time,device,status) values(1509465900000,'wt02',false)", + "insert into wf(time,device,status) values(1509465960000,'wt02',false)", + "insert into wf(time,device,status) values(1509466020000,'wt02',false)", + "insert into wf(time,device,status) values(1509466080000,'wt02',false)", + "insert into wf(time,device,status) values(1509466140000,'wt02',false)", + "insert into wf(time,device,status) values(1509465600000,'wt03',true)", + "insert into wf(time,device,status) values(1509465660000,'wt03',true)", + "insert into wf(time,device,status) values(1509465720000,'wt03',false)", + "insert into wf(time,device,status) values(1509465780000,'wt03',false)", + "insert into wf(time,device,status) values(1509465840000,'wt03',false)", + "insert into wf(time,device,status) values(1509465900000,'wt03',false)", + "insert into wf(time,device,status) values(1509465960000,'wt03',false)", + "insert into wf(time,device,status) values(1509466020000,'wt03',false)", + "insert into wf(time,device,status) values(1509466080000,'wt03',false)", + "insert into wf(time,device,status) values(1509466140000,'wt03',false)", + "insert into wf(time,device,temperature) values(1509465600000,'wt03',25.957603)", + "insert into wf(time,device,temperature) values(1509465660000,'wt03',24.359503)", + "insert into wf(time,device,temperature) values(1509465720000,'wt03',20.092794)", + "insert into wf(time,device,temperature) values(1509465780000,'wt03',20.182663)", + "insert into wf(time,device,temperature) values(1509465840000,'wt03',21.125198)", + "insert into wf(time,device,temperature) values(1509465900000,'wt03',22.720892)", + "insert into wf(time,device,temperature) values(1509465960000,'wt03',20.71)", + "insert into wf(time,device,temperature) values(1509466020000,'wt03',21.451046)", + "insert into wf(time,device,temperature) values(1509466080000,'wt03',22.57987)", + "insert into wf(time,device,temperature) values(1509466140000,'wt03',20.98177)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + + prepareTableData(sqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void selectTest() { + String[] retArray = + new String[] { + defaultFormatDataTime(1509466140000L) + ",wt03,false,20.98177,null,", + defaultFormatDataTime(1509466140000L) + ",wt02,false,null,v1,", + defaultFormatDataTime(1509466140000L) + ",wt01,false,20.98177,null," + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select * from wf where time>1509466080000 order by device desc"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device," + "status," + "temperature,hardware", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void LimitTest() { + String[] retArray = + new String[] { + defaultFormatDataTime(1509466140000L) + ",wt02,false,null,v1,", + defaultFormatDataTime(1509466140000L) + ",wt01,false,20.98177,null," + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select * from wf where time>1509466080000 order by device desc offset 1 limit 2"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device," + "status," + "temperature,hardware", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void InTest() { + String[] retArray = + new String[] { + defaultFormatDataTime(1509465780000L) + ",wt03,false,20.182663,null,", + defaultFormatDataTime(1509465780000L) + ",wt02,false,null,v1,", + defaultFormatDataTime(1509465780000L) + ",wt01,false,20.182663,null," + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select * from wf where time in(1509465780000) order by device desc"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device," + "status," + "temperature,hardware", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + + resultSet = + statement.executeQuery( + "select * from wf where time in(1509465780000) order by device desc"); + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device," + "status," + "temperature,hardware", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.VARCHAR, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } + + @Test + public void testRightTextQuery() { + String[] retArray = + new String[] { + defaultFormatDataTime(1509465600000L) + ",wt02,true,null,v2,", + defaultFormatDataTime(1509465660000L) + ",wt02,true,null,v2," + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery("select * from wf where hardware='v2'"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device," + "status," + "temperature,hardware", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + resultSet = statement.executeQuery("select * from wf where hardware>'v1'"); + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "time,device," + "status," + "temperature,hardware", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, Types.FLOAT, Types.VARCHAR, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void RegexpTest() { + String[] retArray = + new String[] { + "1509465600000,v2,true,", "1509465660000,v2,true,", "1509465720000,v1,false,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + // Matches a string consisting of one lowercase letter and one digit. such as: "v1","v2" + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from wf where hardware regexp '^[a-z][0-9]$' and time < 1509465780000"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "wf.hardware,wf.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + + retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + resultSet = + statement.executeQuery("select hardware,status from wf where hardware regexp 'v*' "); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "wf.hardware,wf.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void RegexpNonExistTest() { + + // Match nonexistent string.'s.' is indicates that the starting with s and the last is any + // single character + String[] retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select hardware,status from wf where hardware regexp 's.' "); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "wf.hardware,wf.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(0, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBQueryWithComplexValueFilterTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBQueryWithComplexValueFilterTableIT.java new file mode 100644 index 0000000000000..e5f495cdf46c9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBQueryWithComplexValueFilterTableIT.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBQueryWithComplexValueFilterTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE sg1(device STRING TAG, s1 INT32 FIELD, s2 DOUBLE FIELD, s3 STRING FIELD, s4 DATE FIELD, s5 TIMESTAMP FIELD)", + "insert into sg1(time,device,s1,s2,s3,s4,s5) values(0,'d1',0,0,'0','2024-01-01',0)", + "insert into sg1(time,device,s1,s2,s3,s4,s5) values(1,'d1',1,1,'1','2024-01-02',1)", + "insert into sg1(time,device,s1,s2) values(2,'d1',2,2)", + "insert into sg1(time,device,s1,s2) values(3,'d1',3,3)", + "insert into sg1(time,device,s1,s2) values(4,'d1',4,4)", + "insert into sg1(time,device,s1,s2) values(5,'d1',5,5)", + "insert into sg1(time,device,s1,s2) values(6,'d1',6,6)", + "insert into sg1(time,device,s1,s2) values(7,'d1',7,7)", + "insert into sg1(time,device,s1,s2) values(8,'d1',8,8)", + "insert into sg1(time,device,s1,s2) values(9,'d1',9,9)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testRawQuery1() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery( + "select s1 from sg1 where (time > 4 and s1 <= 6) or (s2 > 3 and time <= 5)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(3, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRawQuery2() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery( + "select s1 from sg1 where (time > 4 and s1 <= 6) and (s2 > 3 and time <= 5)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRawQuery3() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 = '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 = CAST('2024-01-01' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 = 1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 != '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 != CAST('2024-01-01' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 != 1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRawQuery4() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 > '0'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 > CAST('2024-01-01' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 > 0")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 < '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 < CAST('2024-01-02' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 < 1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 >= '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 >= CAST('2024-01-01' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(2, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 >= 1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 <= '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(2, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 <= CAST('2024-01-01' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 <= 1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(2, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRawQuery5() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 = '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s3 = '1'")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery("select s1 from sg1 where s4 = CAST('2024-01-01' AS DATE)")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + + try (ResultSet resultSet = statement.executeQuery("select s1 from sg1 where s5 = 1")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(1, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java new file mode 100644 index 0000000000000..a7ad4a07be957 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBResultSetTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE t1(device STRING TAG, status BOOLEAN FIELD, temperature FLOAT FIELD, type INT32 FIELD, grade INT64 FIELD)", + "CREATE TABLE sg(device STRING TAG, status FLOAT FIELD)", + }; + + private static final String[] emptyResultSet = new String[] {}; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void intAndLongConversionTest() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "insert into t1(time, device, status, type, grade) values (1000, 'wt01', true, 1, 1000)"); + statement.execute( + "insert into t1(time, device, status, type, grade) values (2000, 'wt01', false, 2, 2000)"); + + // TODO + /*try (ResultSet resultSet1 = + statement.executeQuery("select count(status) from t1")) { + resultSet1.next(); + // type of r1 is INT64(long), test long convert to int + int countStatus = resultSet1.getInt(1); + Assert.assertEquals(2L, countStatus); + } + */ + try (ResultSet resultSet2 = + statement.executeQuery("select type from t1 where time = 1000 limit 1")) { + resultSet2.next(); + // type of r2 is INT32(int), test int convert to long + long type = resultSet2.getLong(1); + Assert.assertEquals(1, type); + } + + try (ResultSet resultSet3 = + statement.executeQuery("select grade from t1 where time = 1000 limit 1")) { + resultSet3.next(); + // type of r3 is INT64(long), test long convert to int + int grade = resultSet3.getInt(1); + Assert.assertEquals(1000, grade); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void columnTypeTest() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + try (ResultSet resultSet = statement.executeQuery("select * from sg")) { + Assert.assertTrue(!resultSet.next()); + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(3, metaData.getColumnCount()); + assertEquals("time", metaData.getColumnName(1)); + assertEquals(Types.TIMESTAMP, metaData.getColumnType(1)); + assertEquals("TIMESTAMP", metaData.getColumnTypeName(1)); + + assertEquals("device", metaData.getColumnName(2)); + assertEquals(Types.VARCHAR, metaData.getColumnType(2)); + assertEquals("STRING", metaData.getColumnTypeName(2)); + + assertEquals("status", metaData.getColumnName(3)); + assertEquals(Types.FLOAT, metaData.getColumnType(3)); + assertEquals("FLOAT", metaData.getColumnTypeName(3)); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void emptyQueryTest1() { + tableAssertTestFail("select * from sg1", "550: Table 'test.sg1' does not exist", DATABASE_NAME); + } + + @Test + public void emptyQueryTest2() { + String[] expectedHeader = + new String[] {"time", "device", "status", "temperature", "type", "grade"}; + tableResultSetEqualTest( + "select * from t1 where false", expectedHeader, emptyResultSet, DATABASE_NAME); + } + + @Test + public void timeWasNullTest() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE test(device STRING TAG, s1 INT64 FIELD, s2 INT64 FIELD, s3 INT64 FIELD)"); + + for (int i = 0; i < 10; i++) { + statement.addBatch( + "insert into test(time, device, s1, s2) values(" + i + ",'d1'," + 1 + "," + 1 + ")"); + } + + statement.execute("insert into test(time, device, s3) values(103,'d1',1)"); + statement.execute("insert into test(time, device, s3) values(104,'d1',1)"); + statement.execute("insert into test(time, device, s3) values(105,'d1',1)"); + statement.executeBatch(); + try (ResultSet resultSet = statement.executeQuery("select * from test")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + while (resultSet.next()) { + for (int i = 1; i <= columnCount; i++) { + int ct = metaData.getColumnType(i); + if (ct == Types.TIMESTAMP) { + if (resultSet.wasNull()) { + fail(); + } + } + } + } + } + } + } + + @Ignore // TODO + @Test + public void emptyLastQueryTest() { + String expectedHeader = + "time" + + "," + + ColumnHeaderConstant.TIMESERIES + + "," + + ColumnHeaderConstant.VALUE + + "," + + ColumnHeaderConstant.DATATYPE + + ","; + resultSetEqualTest("select last s1 from sg", expectedHeader, emptyResultSet); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectCompareExpressionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectCompareExpressionTableIT.java new file mode 100644 index 0000000000000..c8f464b6469f3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectCompareExpressionTableIT.java @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSelectCompareExpressionTableIT { + private static final String DATABASE_NAME = "test"; + private static String[] INSERTION_SQLS; + private static List time = new ArrayList<>(0); + private static List intValue = new ArrayList<>(0); + private static List longValue = new ArrayList<>(0); + private static List floatValue = new ArrayList<>(0); + private static List doubleValue = new ArrayList<>(0); + private static List boolValue = new ArrayList<>(0); + + private static void generateInsertionSQLS() { + INSERTION_SQLS = new String[50]; + Random random = new Random(); + for (int j = 0; j < 50; ++j) { + intValue.add(random.nextInt(10)); + longValue.add((long) random.nextInt(10)); + floatValue.add((float) (random.nextInt(100) / 10.0)); + doubleValue.add(random.nextInt(100) / 10.0); + boolValue.add(random.nextBoolean()); + INSERTION_SQLS[j] = + generateInsertionSQL( + (long) j, + intValue.get(intValue.size() - 1), + longValue.get(longValue.size() - 1), + floatValue.get(floatValue.size() - 1), + doubleValue.get(doubleValue.size() - 1), + boolValue.get(boolValue.size() - 1), + "'magic_words'"); + } + } + + private static String generateInsertionSQL( + long time, + int intValue32, + long intValue64, + float floatValue, + double doubleValue, + boolean boolValue, + String _text) { + return String.format( + Locale.CHINA, + "insert into t1(time, device, s1, s2, s3, s4, s5, s6) values (%d, 'd1', %d, %d, %f, %f, %s, %s)", + time, + intValue32, + intValue64, + floatValue, + doubleValue, + boolValue ? "true" : "false", + _text); + } + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + createTimeSeries(); + generateData(); + } + + private static void generateData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + generateInsertionSQLS(); + statement.execute("USE " + DATABASE_NAME); + for (String dataGenerationSql : INSERTION_SQLS) { + statement.execute(dataGenerationSql); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void createTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE t1(device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD)"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /* + * Test compare expressions between different TSDataType + * */ + @Test + public void testCompareWithConstant() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select time, s1>=5, s1<=5, s1>5, s1<5, s1=5, s1!=5 from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 6, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals(intValue.get(i) >= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(intValue.get(i) <= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(intValue.get(i) > 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(intValue.get(i) < 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(intValue.get(i) == 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(intValue.get(i) != 5, bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select time, s2>=5, s2<=5, s2>5, s2<5, s2=5, s2!=5 from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 6, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals(longValue.get(i) >= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(longValue.get(i) <= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(longValue.get(i) > 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(longValue.get(i) < 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(longValue.get(i) == 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(longValue.get(i) != 5, bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select time, s3>=5, s3<=5, s3>5, s3<5, s3=5, s3!=5 from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 6, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals(floatValue.get(i) >= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(floatValue.get(i) <= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(floatValue.get(i) > 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(floatValue.get(i) < 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(floatValue.get(i) == 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(floatValue.get(i) != 5, bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select time, s4>=5, s4<=5, s4>5, s4<5, s4=5, s4!=5 from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 6, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals(doubleValue.get(i) >= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(doubleValue.get(i) <= 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(doubleValue.get(i) > 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(doubleValue.get(i) < 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(doubleValue.get(i) == 5, bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(doubleValue.get(i) != 5, bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select time, s5=true, s5!=true, s5=false, s5!=false from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 4, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals(boolValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(!boolValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(!boolValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(boolValue.get(i), bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCompareDifferentType() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select time, s1>=s2, s1<=s2, s1>s3, s1= longValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(intValue.get(i) <= longValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(intValue.get(i) > floatValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(intValue.get(i) < floatValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals((double) intValue.get(i) == doubleValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals((double) intValue.get(i) != doubleValue.get(i), bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select time, s2>=s3, s2<=s3, s2>s4, s2= floatValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(longValue.get(i) <= floatValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(longValue.get(i) > doubleValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(longValue.get(i) < doubleValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(longValue.get(i) == (long) intValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(longValue.get(i) != (long) intValue.get(i), bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select time, s3>=s4, s3<=s4, s3>s1, s3= doubleValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(floatValue.get(i) <= doubleValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(floatValue.get(i) > intValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(floatValue.get(i) < intValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(floatValue.get(i) == (float) longValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(floatValue.get(i) != (float) longValue.get(i), bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select time, s4>=s1, s4<=s1, s4>s2, s4= intValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(doubleValue.get(i) <= intValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(doubleValue.get(i) > longValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(5)); + assertEquals(doubleValue.get(i) < longValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(6)); + assertEquals(doubleValue.get(i) == (double) floatValue.get(i), bool); + + bool = Boolean.parseBoolean(resultSet.getString(7)); + assertEquals(doubleValue.get(i) != (double) floatValue.get(i), bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testLogicOrAndNot() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select time, s1>=1 and s1<3, not(s1 < 2 or s1> 8), not(s2>3) from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 3, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals(intValue.get(i) >= 1 && intValue.get(i) < 3, bool); + + bool = Boolean.parseBoolean(resultSet.getString(3)); + assertEquals(!(intValue.get(i) < 2 || intValue.get(i) > 8), bool); + + bool = Boolean.parseBoolean(resultSet.getString(4)); + assertEquals(!(longValue.get(i) > 3), bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testComplexExpression() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + "select time, ( s1 + 1 ) * 2 - 4 < ( s3 * 3 - 6) / 2 and ( s1 + 5 ) * 2 > s2 * 3 + 4 from t1"); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + 1, columnCount); + + boolean bool; + for (int i = 0; i < time.size(); ++i) { + resultSet.next(); + + bool = Boolean.parseBoolean(resultSet.getString(2)); + assertEquals( + (intValue.get(i) + 1) * 2 - 4 < (floatValue.get(i) * 3 - 6) / 2 + && (intValue.get(i) + 5) * 2 > longValue.get(i) * 3 + 4, + bool); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java new file mode 100644 index 0000000000000..e17f873e116b5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSelectSchemaTableIT { + private static final String DATABASE_NAME = "test"; + private static String[] SQLS = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE sg(device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 DOUBLE FIELD)", + "insert into sg(time, device, s1, s2, s3) values (1, 'd1', 1, 2, 3.0)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testSchemaExpression() { + String[] expressions = { + "s1+s2", + "-s1+s2", + "-(s1+s3)", + "not(s1>s2)", + "-(-(s1))", + "((s1+s2)*s3)", + "-2+s1", + "not true or s1>0", + "-(-1)+s1", + "sin(s1)+s1", + "((s1+1)*2-1)%2+1.5+s2" + }; + String[] columnNames = { + "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", "_col10", + "_col11", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery( + String.format( + "select time, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s from sg", + expressions[0], + expressions[1], + expressions[2], + expressions[3], + expressions[4], + expressions[5], + expressions[6], + expressions[7], + expressions[8], + expressions[9], + expressions[10])); + int columnCount = resultSet.getMetaData().getColumnCount(); + assertEquals(1 + expressions.length, columnCount); + + for (int i = 0; i < expressions.length; ++i) { + assertEquals(columnNames[i], resultSet.getMetaData().getColumnName(i + 2).replace(" ", "")); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBColumnsMatchTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBColumnsMatchTableIT.java new file mode 100644 index 0000000000000..e6a3ddae29db9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBColumnsMatchTableIT.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBColumnsMatchTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD)", + "INSERT INTO table1(time,device_id,s1,s2) values(1,'d1',1,10)", + "INSERT INTO table1(time,device_id,s1,s2) values(2,'d2',2,20)", + "INSERT INTO table1(time,device_id,s1) values(3,'d3',3)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void columnsTest() { + + // case 1: match all columns + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,10,", + "1970-01-01T00:00:00.002Z,d2,2,20,", + "1970-01-01T00:00:00.003Z,d3,3,null," + }; + tableResultSetEqualTest( + "SELECT COLUMNS(*) FROM table1 order by time", expectedHeader, retArray, DATABASE_NAME); + + // case 2: match columns which name starts with 's' + expectedHeader = new String[] {"s1", "s2"}; + retArray = new String[] {"1,10,", "2,20,", "3,null,"}; + tableResultSetEqualTest( + "SELECT COLUMNS('^s.*') FROM table1 order by s1", expectedHeader, retArray, DATABASE_NAME); + + // case 3: cannot match any column + tableAssertTestFail( + "SELECT COLUMNS('^b.*') FROM table1", + "701: No matching columns found that match regex '^b.*'", + DATABASE_NAME); + + // case 4: invalid input of regex + tableAssertTestFail( + "SELECT COLUMNS('*b') FROM table1 order by s1", "701: Invalid regex '*b'", DATABASE_NAME); + } + + @Test + public void aliasTest() { + // case 1: normal test + String[] expectedHeader = new String[] {"series1", "series2"}; + String[] retArray = new String[] {"1,10,", "2,20,", "3,null,"}; + tableResultSetEqualTest( + "SELECT COLUMNS('^s(.*)') AS \"series$1\" FROM table1 order by series1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"s", "s", "s", "s"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,10,", + "1970-01-01T00:00:00.002Z,d2,2,20,", + "1970-01-01T00:00:00.003Z,d3,3,null," + }; + tableResultSetEqualTest( + "SELECT COLUMNS(*) AS s FROM table1 order by 1", expectedHeader, retArray, DATABASE_NAME); + + // case 2: no according reference group + tableAssertTestFail( + "SELECT COLUMNS('^s.*') AS \"series$1\" FROM table1", "701: No group 1", DATABASE_NAME); + + // case 3: alias grammar error + tableAssertTestFail( + "SELECT COLUMNS('^s.*') AS 'series$1' FROM table1", + "700: line 1:27: mismatched input ''series$1''. Expecting: ", + DATABASE_NAME); + } + + @Test + public void usedWithExpressionTest1() { + // case 1: select min value for each column + String[] expectedHeader = + new String[] {"_col0_time", "_col1_device_id", "_col2_s1", "_col3_s2"}; + String[] retArray = new String[] {"1970-01-01T00:00:00.001Z,d1,1,10,"}; + tableResultSetEqualTest( + "SELECT min(COLUMNS(*)) FROM table1", expectedHeader, retArray, DATABASE_NAME); + + // case 2: multi columns() in different selectItem, + expectedHeader = + new String[] { + "_col0_time", "_col1_device_id", "_col2_s1", "_col3_s2", "_col4_s1", "_col5_s2" + }; + retArray = new String[] {"1970-01-01T00:00:00.001Z,d1,1,10,1,10,"}; + tableResultSetEqualTest( + "SELECT min(COLUMNS(*)), min(COLUMNS('^s.*')) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: multi columns() in one expression, columns() are same + expectedHeader = new String[] {"_col0_s1", "_col1_s2"}; + retArray = new String[] {"4,30,"}; + tableResultSetEqualTest( + "SELECT min(COLUMNS('^s.*')) + max(COLUMNS('^s.*')) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: multi columns() in one expression, columns() are different + tableAssertTestFail( + "SELECT min(COLUMNS('^s.*')) + max(COLUMNS('^t.*')) FROM table1", + "701: Multiple different COLUMNS in the same expression are not supported", + DATABASE_NAME); + + // case 5: get last row of Data + expectedHeader = + new String[] {"_col0", "_col1_time", "_col2_device_id", "_col3_s1", "_col4_s2"}; + retArray = new String[] {"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,d3,3,null,"}; + tableResultSetEqualTest( + "SELECT last(time), last_by(COLUMNS(*), time) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: get last non-null value of each column, and according time + expectedHeader = + new String[] { + "_col0_time", + "_col1_device_id", + "_col2_s1", + "_col3_s2", + "_col4_time", + "_col5_device_id", + "_col6_s1", + "_col7_s2" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d3,3,20,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.002Z," + }; + tableResultSetEqualTest( + "SELECT last(COLUMNS(*)), last_by(time,COLUMNS(*)) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "test", "test", "test", "test", "_col4_time", "_col5_device_id", "_col6_s1", "_col7_s2" + }; + tableResultSetEqualTest( + "SELECT last(COLUMNS(*)) as test, last_by(time,COLUMNS(*)) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // used with all supported Expressions + @Test + public void usedWithExpressionTest2() { + String[] expectedHeader = + new String[] { + "_col0_s1", + "_col1_s2", + "_col2_s1", + "_col3_s2", + "_col4_s1", + "_col5_s2", + "_col6_s1", + "_col7_s2" + }; + String[] retArray = + new String[] { + "2,20,-1,-10,true,false,false,true,", + "4,40,-2,-20,true,false,true,true,", + "6,null,-3,null,false,null,true,false," + }; + tableResultSetEqualTest( + "select columns('s.*')+columns('s.*'), -columns('s.*'),columns('s.*') between 1 and 2,if(columns('s.*')>1,true,false) from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "select columns(*).s1 from table1", + "701: Columns are not supported in DereferenceExpression", + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + retArray = new String[] {"1970-01-01T00:00:00.002Z,d2,2,20,"}; + tableResultSetEqualTest( + "select * from table1 where columns('s.*') > any(select s1 from table1) order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0_s1", + "_col1_s2", + "_col2_s1", + "_col3_time", + "_col4_device_id", + "_col5_s1", + "_col6_s2", + "_col7_time", + "_col8_device_id", + "_col9_s1", + "_col10_s2", + "_col11_time", + "_col12_device_id", + "_col13_s1", + "_col14_s2" + }; + retArray = + new String[] { + "1,10,1,true,true,true,true,true,true,true,true,false,false,false,false,", + "2,20,2,true,true,true,true,true,true,true,true,false,false,false,false,", + "3,null,3,true,true,true,null,true,true,true,false,false,false,false,true," + }; + tableResultSetEqualTest( + "select trim(cast(columns('s.*') as String)),COALESCE(columns('s1.*'),1), columns(*) in (columns(*)),columns(*) is not null, columns(*) is null from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0_device_id", "_col1_time", "_col2_device_id", "_col3_s1", "_col4_s2"}; + retArray = + new String[] { + "true,true,true,true,true,", "true,true,true,true,true,", "true,true,true,true,null," + }; + tableResultSetEqualTest( + "select columns('d.*') like 'd_',columns(*)=columns(*) from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "SELECT CASE columns('s.*') WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'many' END from table1", + "701: CASE operand type does not match WHEN clause operand type: INT64 vs INT32", + DATABASE_NAME); + + expectedHeader = new String[] {"_col0_s1"}; + retArray = new String[] {"one,", "two,", "many,"}; + tableResultSetEqualTest( + "SELECT CASE columns('s1.*') WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'many' END from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0_s1", "_col1_s2"}; + retArray = new String[] {"one,many,", "two,many,", "many,many,"}; + tableResultSetEqualTest( + "select case when columns('s.*') = 1 then 'one' when columns('s.*')=2 then 'two' else 'many' end from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void usedInWhereTest() { + // case 1: normal test + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + String[] retArray = new String[] {"1970-01-01T00:00:00.002Z,d2,2,20,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE COLUMNS('^s.*') > 1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + retArray = + new String[] {"1970-01-01T00:00:00.002Z,d2,2,20,", "1970-01-01T00:00:00.003Z,d3,3,null,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE COLUMNS('^s1.*') > 1 order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: cannot match any column + tableAssertTestFail( + "SELECT * FROM table1 WHERE COLUMNS('^b.*') > 1", + "701: No matching columns found that match regex '^b.*'", + DATABASE_NAME); + + // case 4: invalid input of regex + tableAssertTestFail( + "SELECT * FROM table1 WHERE COLUMNS('*b') > 1", "701: Invalid regex '*b'", DATABASE_NAME); + } + + @Test + public void otherExceptionTest() { + // cannot be used in HAVING clause + tableAssertTestFail( + "SELECT device_id, count(s1) FROM table1 HAVING COLUMNS('device_id') > 1", + "701: Columns only support to be used in SELECT and WHERE clause", + DATABASE_NAME); + + // cannot be used for columns without name + tableAssertTestFail( + "SELECT COLUMNS(*) FROM (SELECT s1,s1+1 from table1)", + "701: Unknown ColumnName", + DATABASE_NAME); + tableAssertTestFail( + "SELECT COLUMNS('s1') FROM (SELECT s1,s1+1 from table1)", + "701: Unknown ColumnName", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java new file mode 100644 index 0000000000000..ac93e29302e22 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBComplexQueryIT { + protected static final String DATABASE_NAME = "test_db"; + protected static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table employees(department_id STRING TAG,remark STRING ATTRIBUTE,name TEXT FIELD,Gender TEXT FIELD,Status BOOLEAN FIELD,employee_id INT32 FIELD,salary DOUBLE FIELD,date_of_birth DATE FIELD,Contac_info string FIELD)", + "create table departments(department_id STRING TAG,dep_description STRING ATTRIBUTE,dep_name TEXT FIELD,dep_phone TEXT FIELD,dep_status BOOLEAN FIELD,dep_member INT32 FIELD,employee_id INT32 FIELD)", + "insert into employees(time, department_id, remark, name, gender, status, employee_id, salary, date_of_birth, contac_info) values(1, 'D001', 'good', 'Mary','Female', false, 1223, 5500.22, '1988-10-12', '133-1212-1234')", + "insert into employees(time, department_id, remark, name, gender, status, employee_id, salary, date_of_birth, contac_info) values(2, 'D001', 'great', 'John', 'Male', true, 40012, 8822, '1985-06-15', '130-1002-1334')", + "insert into employees(time, department_id, remark, name, gender, status, employee_id, salary, date_of_birth, contac_info) values(3, 'D002', 'excellent', 'Nancy', 'Female', true, 30112, 10002, '1983-08-15', '135-1302-1354')", + "insert into employees(time, department_id, remark, name, gender, status, employee_id, salary, date_of_birth, contac_info) values(4, 'D002', 'good', 'Jack', 'Male', false, 12212, 7000, '1990-03-26', '138-1012-1353')", + "insert into employees(time, department_id, remark, name, gender, status, employee_id, salary, date_of_birth, contac_info) values(5, 'D003', 'great', 'Linda', 'Female', false, 10212, 5600, '1995-06-15', '150-2003-1355')", + "insert into departments(time, department_id, dep_description, dep_name, dep_phone, dep_status, dep_member,employee_id) values(1, 'D001', 'goods','销售部', '010-2271-2120', false, 1223,1223)", + "insert into departments(time, department_id, dep_description, dep_name, dep_phone, dep_status, dep_member,employee_id) values(2, 'D001', 'goods','销售部', '010-2271-2120', false, 102, 40012)", + "insert into departments(time, department_id, dep_description, dep_name, dep_phone, dep_status, dep_member,employee_id) values(3, 'D002', 'service','客服部', '010-2077-2520', true, 220, 30112)", + "insert into departments(time, department_id, dep_description, dep_name, dep_phone, dep_status, dep_member,employee_id) values(4, 'D002', 'service','客服部', '010-2077-2520', true, 2012, 12212)", + "insert into departments(time, department_id, dep_description, dep_name, dep_phone, dep_status, dep_member,employee_id) values(5, 'D003', 'IT','研发部', '010-3272-2310', true, 300, 10212)", + "insert into departments(time, department_id, dep_description, dep_name, dep_phone, dep_status, dep_member,employee_id) values(6, 'D004', 'IT','人事部', '010-3272-2312', true, 300, 10200)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void queryTest1() { + // Look for the non-intersecting departments in the two tables + String[] expectedHeader = new String[] {"department_id", "dep_name"}; + String[] retArray = new String[] {"D004,人事部,"}; + tableResultSetEqualTest( + "select department_id, dep_name from departments where not exists(" + + "select 1 from employees where employees.department_id = departments.department_id)", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java new file mode 100644 index 0000000000000..f0d0ca89ff68c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDistinctTagIT { + private static final String DATABASE_NAME = "test"; + + protected static final String[] SQLs = + new String[] { + "CREATE DATABASE IF NOT EXISTS test", + "USE test", + // test flush + "CREATE TABLE IF NOT EXISTS t1(deviceId STRING TAG, attr1 STRING ATTRIBUTE, s1 INT64 FIELD)", + "insert into t1(time, deviceId, attr1, s1) values(1000, 'd1', 'a1', 10)", + "insert into t1(time, deviceId, attr1, s1) values(1000, 'd2', 'a2', 11)", + "insert into t1(time, deviceId, attr1, s1) values(2000, 'd2', 'xx', 12)", + "insert into t1(time, deviceId, attr1, s1) values(4000, 'd1', 'a1', 13)", + "flush", + "insert into t1(time, deviceId, attr1, s1) values(5000, 'd3', 'a3', 10)", + "insert into t1(time, deviceId, attr1, s1) values(6000, 'd4', 'a4', 11)", + "insert into t1(time, deviceId, attr1, s1) values(3000, 'd2', 'a2', 12)", + "insert into t1(time, deviceId, attr1, s1) values(2000, 'd1', 'a1', 13)", + "flush", + + // test memory + "CREATE TABLE IF NOT EXISTS t2(deviceId STRING TAG, attr1 STRING ATTRIBUTE, s1 INT64 FIELD)", + "insert into t2(time, deviceId, attr1, s1) values(1000, 'd1', 'a1', 10)", + "insert into t2(time, deviceId, attr1, s1) values(1000, 'd2', 'a2', 11)", + "insert into t2(time, deviceId, attr1, s1) values(2000, 'd2', 'xx', 12)", + "insert into t2(time, deviceId, attr1, s1) values(4000, 'd1', 'a1', 13)", + "insert into t2(time, deviceId, attr1, s1) values(5000, 'd3', 'a3', 10)", + "insert into t2(time, deviceId, attr1, s1) values(6000, 'd4', 'a4', 11)", + "insert into t2(time, deviceId, attr1, s1) values(3000, 'd2', 'a2', 12)", + "insert into t2(time, deviceId, attr1, s1) values(2000, 'd1', 'a1', 13)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDistinct() { + // distinct(deviceId) + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {"d1,", "d2,", "d3,", "d4,"}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t1 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(deviceId) from test.t2 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + // distinct(attr1) + expectedHeader = new String[] {"attr1"}; + retArray = new String[] {"a1,", "a2,", "a3,", "a4,"}; + tableResultSetEqualTest( + "select distinct(attr1) from test.t1 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(attr1) from test.t2 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDistinctWithTimeFilter() { + // distinct(deviceId) ... where time > 3000; + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {"d1,", "d3,", "d4,"}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t1 where time > 3000 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(deviceId) from test.t2 where time > 3000 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + // distinct(attr1) ... where time = 5000; + expectedHeader = new String[] {"attr1"}; + retArray = new String[] {"a3,"}; + tableResultSetEqualTest( + "select distinct(attr1) from test.t1 where time = 5000 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(attr1) from test.t2 where time = 5000 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDistinctWithPushDownFilter() { + // distinct(deviceId) ... where s1 = 11; + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {"d2,", "d4,"}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t1 where s1 = 11 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(deviceId) from test.t2 where s1 = 11 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + // distinct(attr1) ... where s1 > 11; + expectedHeader = new String[] {"attr1"}; + retArray = new String[] {"a1,", "a2,"}; + tableResultSetEqualTest( + "select distinct(attr1) from test.t1 where s1 > 11 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(attr1) from test.t2 where s1 > 11 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDistinctWithDelete() throws SQLException { + String[] sqls = + new String[] { + "USE test", + // test flush + "CREATE TABLE IF NOT EXISTS t3(deviceId STRING TAG, attr1 STRING ATTRIBUTE, s1 INT64 FIELD)", + "insert into t3(time, deviceId, attr1, s1) values(1000, 'd1', 'a1', 10)", + "insert into t3(time, deviceId, attr1, s1) values(1000, 'd2', 'a2', 11)", + "insert into t3(time, deviceId, attr1, s1) values(2000, 'd2', 'xx', 12)", + "insert into t3(time, deviceId, attr1, s1) values(4000, 'd1', 'a1', 13)", + "flush", + "delete from test.t3", + + // test memory + "CREATE TABLE IF NOT EXISTS t4(deviceId STRING TAG, attr1 STRING ATTRIBUTE, s1 INT64 FIELD)", + "insert into t4(time, deviceId, attr1, s1) values(1000, 'd1', 'a1', 10)", + "insert into t4(time, deviceId, attr1, s1) values(1000, 'd2', 'a2', 11)", + "insert into t4(time, deviceId, attr1, s1) values(2000, 'd2', 'xx', 12)", + "insert into t4(time, deviceId, attr1, s1) values(4000, 'd1', 'a1', 13)", + "delete from test.t4", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.execute(sql); + } + } + + // distinct(deviceId) + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t3 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct(deviceId) from test.t4 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + // distinct(attr1) + expectedHeader = new String[] {"attr1"}; + retArray = new String[] {}; + tableResultSetEqualTest( + "select distinct(attr1) from test.t3 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(attr1) from test.t4 order by attr1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + private static void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java new file mode 100644 index 0000000000000..3884b1f2be13d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java @@ -0,0 +1,725 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFillTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " values(1, 'd1', 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(2, 'd1', 2, 22, 2.2, 22.2, false)", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(3, 'd1', 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + "flush", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(7, 'd1', 7, 77, 7.7, 77.7, true)", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(8, 'd1', 'text8', 'string8', X'cafebabe08', 8, '2024-10-08')", + "CREATE TABLE table2(city STRING TAG, device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD)", + "INSERT INTO table2(time,city,device_id,s2) values(1, 'shanghai', 'd1', 02111)", + "INSERT INTO table2(time,city,device_id,s1) values(2, 'shanghai', 'd1', 0212)", + "INSERT INTO table2(time,city,device_id,s2) values(1, 'beijing', 'd1', 01011)", + "INSERT INTO table2(time,city,device_id,s1) values(2, 'beijing', 'd1', 0102)", + "INSERT INTO table2(time,city,device_id,s1) values(3, 'beijing', 'd1', 0103)", + "INSERT INTO table2(time,city,device_id,s1) values(4, 'beijing', 'd1', 0104)", + "INSERT INTO table2(time,city,device_id,s1) values(1, 'beijing', 'd2', 0101)", + "INSERT INTO table2(time,city,device_id,s2) values(2, 'beijing', 'd2', 01022)", + "INSERT INTO table2(time,city,device_id,s2) values(3, 'beijing', 'd2', 01033)", + "INSERT INTO table2(time,city,device_id,s2) values(4, 'beijing', 'd2', 01044)", + "CREATE TABLE table3(city STRING TAG, device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD)", + "INSERT INTO table3(time,city,device_id,s2) values(1, 'shanghai', 'd1', 02111)", + "INSERT INTO table3(time,city,device_id,s1) values(2, 'shanghai', 'd1', 0212)", + "INSERT INTO table3(time,city,device_id,s2) values(1, 'beijing', 'd1', 01011)", + "INSERT INTO table3(time,city,device_id,s1) values(2, 'beijing', 'd1', 0102)", + "INSERT INTO table3(time,city,device_id,s1,s2) values(3, 'beijing', 'd1', 0103,01033)", + "INSERT INTO table3(time,city,device_id,s1) values(4, 'beijing', 'd1', 0104)", + "INSERT INTO table3(time,city,device_id,s1) values(1, 'beijing', 'd2', 0101)", + "INSERT INTO table3(time,city,device_id,s2) values(2, 'beijing', 'd2', 01022)", + "INSERT INTO table3(time,city,device_id,s1,s2) values(3, 'beijing', 'd2', 0103, 01033)", + "INSERT INTO table3(time,city,device_id,s2) values(4, 'beijing', 'd2', 01044)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void normalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS", expectedHeader, retArray, DATABASE_NAME); + + // case 2: all with time filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: all without time filter using previous fill with timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: all without time filter using previous fill with timeDuration with helper column + // index + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: all without time filter using previous fill with timeDuration with helper column + // index + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,1,11,1.1,11.1,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 11", + expectedHeader, + retArray, + DATABASE_NAME); + + // case7: all without time filter using previous fill with order by time desc + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1 order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using previous fill with order by value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1 order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using previous fill with subQuery + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.003Z,d1,5,55,5.5,55.5,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select * from table1 order by time desc) FILL METHOD PREVIOUS TIME_BOUND 2ms order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // case10: all without time filter using previous fill with FILL_GROUP + expectedHeader = new String[] {"time", "city", "device_id", "s1", "s2"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,beijing,d1,null,1011,", + "1970-01-01T00:00:00.002Z,beijing,d1,102,1011,", + "1970-01-01T00:00:00.003Z,beijing,d1,103,1011,", + "1970-01-01T00:00:00.004Z,beijing,d1,104,1011,", + "1970-01-01T00:00:00.001Z,beijing,d2,101,null,", + "1970-01-01T00:00:00.002Z,beijing,d2,101,1022,", + "1970-01-01T00:00:00.003Z,beijing,d2,101,1033,", + "1970-01-01T00:00:00.004Z,beijing,d2,101,1044,", + "1970-01-01T00:00:00.001Z,shanghai,d1,null,2111,", + "1970-01-01T00:00:00.002Z,shanghai,d1,212,2111,", + }; + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case11: all without time filter using previous fill with FILL_GROUP and TIME_COLUMN + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS TIME_COLUMN 1 FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case12: all without time filter using previous fill with TIME_BOUND and FILL_GROUP + expectedHeader = new String[] {"time", "city", "device_id", "s1", "s2"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,beijing,d1,null,1011,", + "1970-01-01T00:00:00.002Z,beijing,d1,102,1011,", + "1970-01-01T00:00:00.003Z,beijing,d1,103,1011,", + "1970-01-01T00:00:00.004Z,beijing,d1,104,null,", + "1970-01-01T00:00:00.001Z,beijing,d2,101,null,", + "1970-01-01T00:00:00.002Z,beijing,d2,101,1022,", + "1970-01-01T00:00:00.003Z,beijing,d2,101,1033,", + "1970-01-01T00:00:00.004Z,beijing,d2,null,1044,", + "1970-01-01T00:00:00.001Z,shanghai,d1,null,2111,", + "1970-01-01T00:00:00.002Z,shanghai,d1,212,2111,", + }; + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS TIME_BOUND 2ms FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case13: all without time filter using previous fill with TIME_BOUND, FILL_GROUP and + // TIME_COLUMN + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1 FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + // case 1: all without time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: all with time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 FILL METHOD LINEAR", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL METHOD LINEAR", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: all without time filter using linear fill with helper column + // index + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR TIME_COLUMN 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: all without time filter using linear fill with helper column index + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR TIME_COLUMN 10", + expectedHeader, + retArray, + DATABASE_NAME); + + // case7: all without time filter using linear fill with order by time desc + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using linear fill with order by value + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR TIME_COLUMN 1 order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using linear fill with subQuery + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 order by time desc) FILL METHOD LINEAR order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // case10: all without time filter using linear fill with FILL_GROUP + expectedHeader = new String[] {"time", "city", "device_id", "s1", "s2"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,beijing,d1,null,1011,", + "1970-01-01T00:00:00.002Z,beijing,d1,102,1022,", + "1970-01-01T00:00:00.003Z,beijing,d1,103,1033,", + "1970-01-01T00:00:00.004Z,beijing,d1,104,null,", + "1970-01-01T00:00:00.001Z,beijing,d2,101,null,", + "1970-01-01T00:00:00.002Z,beijing,d2,102,1022,", + "1970-01-01T00:00:00.003Z,beijing,d2,103,1033,", + "1970-01-01T00:00:00.004Z,beijing,d2,null,1044,", + "1970-01-01T00:00:00.001Z,shanghai,d1,null,2111,", + "1970-01-01T00:00:00.002Z,shanghai,d1,212,null,", + }; + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table3 FILL METHOD LINEAR FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case11: all without time filter using linear fill with FILL_GROUP and TIME_COLUMN + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table3 FILL METHOD LINEAR TIME_COLUMN 1 FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- VALUE FILL --------------------------------- + // case 1: fill with integer value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.003Z,d1,100,100,100.0,100.0,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,100,100,100.0,100.0,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.008Z,d1,100,100,100.0,100.0,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT 100", expectedHeader, retArray, DATABASE_NAME); + + // case 2: fill with float value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.003Z,d1,110,110,110.2,110.2,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,110,110,110.2,110.2,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.008Z,d1,110,110,110.2,110.2,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT 110.2", expectedHeader, retArray, DATABASE_NAME); + + // case 3: fill with boolean value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,false,false,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,0,0,0.0,0.0,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,0,0,0.0,0.0,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,false,false,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,false,false,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,0,0,0.0,0.0,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT false", expectedHeader, retArray, DATABASE_NAME); + + // case 4: fill with string value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT 'iotdb'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT '2018-05-06'", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: fill with blob value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT X'cafebabe99'", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void abNormalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + tableAssertTestFail( + "select s1 from table1 FILL METHOD PREVIOUS TIME_COLUMN 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Don't need to specify TIME_COLUMN while either TIME_BOUND or FILL_GROUP parameter is not specified", + DATABASE_NAME); + + tableAssertTestFail( + "select s1 from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer TIME_COLUMN for PREVIOUS FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1 from table1 FILL METHOD PREVIOUS FILL_GROUP 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer TIME_COLUMN for PREVIOUS FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of TIME_COLUMN for PREVIOUS FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL TIME_COLUMN position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL TIME_COLUMN position 3 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS FILL_GROUP 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL FILL_GROUP position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS FILL_GROUP 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL FILL_GROUP position 3 is not in select list", + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + tableAssertTestFail( + "select s1 from table1 FILL METHOD LINEAR", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer TIME_COLUMN for LINEAR FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR TIME_COLUMN 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of TIME_COLUMN for LINEAR FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR TIME_COLUMN 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL TIME_COLUMN position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR TIME_COLUMN 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL TIME_COLUMN position 3 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR FILL_GROUP 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL FILL_GROUP position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR FILL_GROUP 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL FILL_GROUP position 3 is not in select list", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java new file mode 100644 index 0000000000000..3c8a6f3af802b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java @@ -0,0 +1,547 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBGapFillTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(city STRING TAG, device_id STRING TAG, s1 DOUBLE FIELD, s2 INT64 FIELD)", + "INSERT INTO table1(time,city,device_id,s2) values(2024-09-24T06:15:46.565+00:00, 'shanghai', 'd1', 2)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T07:16:15.297+00:00, 'shanghai', 'd1', 27.2)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T08:16:21.907+00:00, 'shanghai', 'd1', 27.3)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T11:16:28.821+00:00, 'shanghai', 'd1', 29.3)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T04:14:40.545+00:00, 'beijing', 'd2', 25.1)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T11:17:15.297+00:00, 'beijing', 'd2', 28.2)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T08:15:21.907+00:00, 'shanghai', 'd3', 22.3)", + "INSERT INTO table1(time,city,device_id,s1) values(2024-09-24T08:15:28.821+00:00, 'shanghai', 'd3', 29.3)", + }; + + private static final String MULTI_GAFILL_ERROR_MSG = + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": multiple date_bin_gapfill calls not allowed"; + private static final String TIME_RANGE_CANNOT_INFER_ERROR_MSG = + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": could not infer startTime or endTime from WHERE clause"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void normalGapFillTest() { + + // case 1: avg_s1 of one device without having + String[] expectedHeader = new String[] {"hour_time", "avg_s1"}; + String[] retArray = + new String[] { + "2024-09-24T04:00:00.000Z,null,", + "2024-09-24T05:00:00.000Z,null,", + "2024-09-24T06:00:00.000Z,null,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,27.3,", + "2024-09-24T09:00:00.000Z,null,", + "2024-09-24T10:00:00.000Z,null,", + "2024-09-24T11:00:00.000Z,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,city,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: avg_s1 of one device with having + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 HAVING avg(s1) IS NOT NULL", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,city,device_id HAVING avg(s1) IS NOT NULL", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: avg_s1 of each device + expectedHeader = new String[] {"hour_time", "city", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T05:00:00.000Z,beijing,d2,null,", + "2024-09-24T06:00:00.000Z,beijing,d2,null,", + "2024-09-24T07:00:00.000Z,beijing,d2,null,", + "2024-09-24T08:00:00.000Z,beijing,d2,null,", + "2024-09-24T09:00:00.000Z,beijing,d2,null,", + "2024-09-24T10:00:00.000Z,beijing,d2,null,", + "2024-09-24T11:00:00.000Z,beijing,d2,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,d1,null,", + "2024-09-24T05:00:00.000Z,shanghai,d1,null,", + "2024-09-24T06:00:00.000Z,shanghai,d1,null,", + "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T09:00:00.000Z,shanghai,d1,null,", + "2024-09-24T10:00:00.000Z,shanghai,d1,null,", + "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,", + "2024-09-24T04:00:00.000Z,shanghai,d3,null,", + "2024-09-24T05:00:00.000Z,shanghai,d3,null,", + "2024-09-24T06:00:00.000Z,shanghai,d3,null,", + "2024-09-24T07:00:00.000Z,shanghai,d3,null,", + "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T09:00:00.000Z,shanghai,d3,null,", + "2024-09-24T10:00:00.000Z,shanghai,d3,null,", + "2024-09-24T11:00:00.000Z,shanghai,d3,null,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,device_id order by city,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time),city,device_id order by city,device_id,date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 order by 2,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: avg_s1 of all devices + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,25.1,", + "2024-09-24T05:00:00.000Z,null,", + "2024-09-24T06:00:00.000Z,null,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,26.3,", + "2024-09-24T09:00:00.000Z,null,", + "2024-09-24T10:00:00.000Z,null,", + "2024-09-24T11:00:00.000Z,28.75,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: avg_s1 of all cities + expectedHeader = new String[] {"hour_time", "city", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,25.1,", + "2024-09-24T05:00:00.000Z,beijing,null,", + "2024-09-24T06:00:00.000Z,beijing,null,", + "2024-09-24T07:00:00.000Z,beijing,null,", + "2024-09-24T08:00:00.000Z,beijing,null,", + "2024-09-24T09:00:00.000Z,beijing,null,", + "2024-09-24T10:00:00.000Z,beijing,null,", + "2024-09-24T11:00:00.000Z,beijing,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,null,", + "2024-09-24T05:00:00.000Z,shanghai,null,", + "2024-09-24T06:00:00.000Z,shanghai,null,", + "2024-09-24T07:00:00.000Z,shanghai,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,26.3,", + "2024-09-24T09:00:00.000Z,shanghai,null,", + "2024-09-24T10:00:00.000Z,shanghai,null,", + "2024-09-24T11:00:00.000Z,shanghai,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: avg_s1 of all device_ids + expectedHeader = new String[] {"hour_time", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,d1,null,", + "2024-09-24T05:00:00.000Z,d1,null,", + "2024-09-24T06:00:00.000Z,d1,null,", + "2024-09-24T07:00:00.000Z,d1,27.2,", + "2024-09-24T08:00:00.000Z,d1,27.3,", + "2024-09-24T09:00:00.000Z,d1,null,", + "2024-09-24T10:00:00.000Z,d1,null,", + "2024-09-24T11:00:00.000Z,d1,29.3,", + "2024-09-24T04:00:00.000Z,d2,25.1,", + "2024-09-24T05:00:00.000Z,d2,null,", + "2024-09-24T06:00:00.000Z,d2,null,", + "2024-09-24T07:00:00.000Z,d2,null,", + "2024-09-24T08:00:00.000Z,d2,null,", + "2024-09-24T09:00:00.000Z,d2,null,", + "2024-09-24T10:00:00.000Z,d2,null,", + "2024-09-24T11:00:00.000Z,d2,28.2,", + "2024-09-24T04:00:00.000Z,d3,null,", + "2024-09-24T05:00:00.000Z,d3,null,", + "2024-09-24T06:00:00.000Z,d3,null,", + "2024-09-24T07:00:00.000Z,d3,null,", + "2024-09-24T08:00:00.000Z,d3,25.8,", + "2024-09-24T09:00:00.000Z,d3,null,", + "2024-09-24T10:00:00.000Z,d3,null,", + "2024-09-24T11:00:00.000Z,d3,null,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 7: no data after where + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = new String[] {}; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T00:00:00.000+00:00 AND time < 2024-09-24T06:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void gapFillWithFillClauseTest() { + + // case 1: avg_s1 of one device without having + String[] expectedHeader = new String[] {"hour_time", "avg_s1"}; + String[] retArray = + new String[] { + "2024-09-24T04:00:00.000Z,null,", + "2024-09-24T05:00:00.000Z,null,", + "2024-09-24T06:00:00.000Z,null,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,27.3,", + "2024-09-24T09:00:00.000Z,27.3,", + "2024-09-24T10:00:00.000Z,27.3,", + "2024-09-24T11:00:00.000Z,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,city,device_id FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: avg_s1 of one device with having + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 HAVING avg(s1) IS NOT NULL FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,device_id,city HAVING avg(s1) IS NOT NULL FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: avg_s1 of each device + expectedHeader = new String[] {"hour_time", "city", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T05:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T06:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T07:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T08:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T09:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T10:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T11:00:00.000Z,beijing,d2,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,d1,null,", + "2024-09-24T05:00:00.000Z,shanghai,d1,null,", + "2024-09-24T06:00:00.000Z,shanghai,d1,null,", + "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T09:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T10:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,", + "2024-09-24T04:00:00.000Z,shanghai,d3,null,", + "2024-09-24T05:00:00.000Z,shanghai,d3,null,", + "2024-09-24T06:00:00.000Z,shanghai,d3,null,", + "2024-09-24T07:00:00.000Z,shanghai,d3,null,", + "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T09:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T10:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T11:00:00.000Z,shanghai,d3,25.8,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,device_id FILL METHOD PREVIOUS FILL_GROUP 2,3 order by city,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time),city,device_id FILL METHOD PREVIOUS FILL_GROUP 2,3 order by city,device_id,date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,shanghai,d1,null,", + "2024-09-24T05:00:00.000Z,shanghai,d1,null,", + "2024-09-24T06:00:00.000Z,shanghai,d1,null,", + "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T09:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T10:00:00.000Z,shanghai,d1,null,", + "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,", + "2024-09-24T04:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T05:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T06:00:00.000Z,beijing,d2,null,", + "2024-09-24T07:00:00.000Z,beijing,d2,null,", + "2024-09-24T08:00:00.000Z,beijing,d2,null,", + "2024-09-24T09:00:00.000Z,beijing,d2,null,", + "2024-09-24T10:00:00.000Z,beijing,d2,null,", + "2024-09-24T11:00:00.000Z,beijing,d2,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,d3,null,", + "2024-09-24T05:00:00.000Z,shanghai,d3,null,", + "2024-09-24T06:00:00.000Z,shanghai,d3,null,", + "2024-09-24T07:00:00.000Z,shanghai,d3,null,", + "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T09:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T10:00:00.000Z,shanghai,d3,null,", + "2024-09-24T11:00:00.000Z,shanghai,d3,null,", + }; + // with time bound + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 FILL METHOD PREVIOUS TIME_BOUND 1h FILL_GROUP 2,3 order by 3,2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: avg_s1 of all devices + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,25.1,", + "2024-09-24T05:00:00.000Z,25.1,", + "2024-09-24T06:00:00.000Z,25.1,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,26.3,", + "2024-09-24T09:00:00.000Z,26.3,", + "2024-09-24T10:00:00.000Z,26.3,", + "2024-09-24T11:00:00.000Z,28.75,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 FILL METHOD PREVIOUS order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) FILL METHOD PREVIOUS order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 FILL METHOD PREVIOUS order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) FILL METHOD PREVIOUS order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: avg_s1 of all cities + expectedHeader = new String[] {"hour_time", "city", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,25.1,", + "2024-09-24T05:00:00.000Z,beijing,25.1,", + "2024-09-24T06:00:00.000Z,beijing,25.1,", + "2024-09-24T07:00:00.000Z,beijing,25.1,", + "2024-09-24T08:00:00.000Z,beijing,25.1,", + "2024-09-24T09:00:00.000Z,beijing,25.1,", + "2024-09-24T10:00:00.000Z,beijing,25.1,", + "2024-09-24T11:00:00.000Z,beijing,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,null,", + "2024-09-24T05:00:00.000Z,shanghai,null,", + "2024-09-24T06:00:00.000Z,shanghai,null,", + "2024-09-24T07:00:00.000Z,shanghai,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,26.3,", + "2024-09-24T09:00:00.000Z,shanghai,26.3,", + "2024-09-24T10:00:00.000Z,shanghai,26.3,", + "2024-09-24T11:00:00.000Z,shanghai,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 FILL METHOD PREVIOUS TIME_COLUMN 1 FILL_GROUP 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 FILL METHOD PREVIOUS FILL_GROUP 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: avg_s1 of all device_ids + expectedHeader = new String[] {"hour_time", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,d1,null,", + "2024-09-24T05:00:00.000Z,d1,null,", + "2024-09-24T06:00:00.000Z,d1,null,", + "2024-09-24T07:00:00.000Z,d1,27.2,", + "2024-09-24T08:00:00.000Z,d1,27.3,", + "2024-09-24T09:00:00.000Z,d1,27.3,", + "2024-09-24T10:00:00.000Z,d1,27.3,", + "2024-09-24T11:00:00.000Z,d1,29.3,", + "2024-09-24T04:00:00.000Z,d2,25.1,", + "2024-09-24T05:00:00.000Z,d2,25.1,", + "2024-09-24T06:00:00.000Z,d2,25.1,", + "2024-09-24T07:00:00.000Z,d2,25.1,", + "2024-09-24T08:00:00.000Z,d2,25.1,", + "2024-09-24T09:00:00.000Z,d2,25.1,", + "2024-09-24T10:00:00.000Z,d2,25.1,", + "2024-09-24T11:00:00.000Z,d2,28.2,", + "2024-09-24T04:00:00.000Z,d3,null,", + "2024-09-24T05:00:00.000Z,d3,null,", + "2024-09-24T06:00:00.000Z,d3,null,", + "2024-09-24T07:00:00.000Z,d3,null,", + "2024-09-24T08:00:00.000Z,d3,25.8,", + "2024-09-24T09:00:00.000Z,d3,25.8,", + "2024-09-24T10:00:00.000Z,d3,25.8,", + "2024-09-24T11:00:00.000Z,d3,25.8,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 FILL METHOD PREVIOUS FILL_GROUP 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 7: no data after where + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = new String[] {}; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T00:00:00.000+00:00 AND time < 2024-09-24T06:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void abNormalGapFillTest() { + + // case 1: multiple date_bin_gapfill in group by clause + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where device_id = 'd1' group by date_bin_gapfill(1s, time), date_bin_gapfill(2s, time), 2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + MULTI_GAFILL_ERROR_MSG, + DATABASE_NAME); + + // case 2: no time filter + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 3: with or + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.000+00:00) OR device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 4: time filter is not between + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where (time not between 2024-09-24T04:00:00.000+00:00 AND 2024-09-24T12:00:00.000+00:00) AND (device_id NOT IN ('d3')) group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 5: only > + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time > 2024-09-24T04:00:00.000+00:00 AND device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 6: only >= + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time >= 2024-09-24T04:00:00.000+00:00 group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 7: only < + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time < 2024-09-24T12:00:00.000+00:00 AND device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 8: only <= + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time <= 2024-09-24T12:00:00.000+00:00 group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java new file mode 100644 index 0000000000000..fd5d0d94e01ca --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.auth.AuthorityChecker.ONLY_ADMIN_ALLOWED; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableQueryNoVerifyResultTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBMaintainAuthIT { + private static final String DATABASE_NAME = "test"; + private static final String CREATE_USER_FORMAT = "create user %s '%s'"; + private static final String USER_1 = "user1"; + private static final String USER_2 = "user2"; + private static final String PASSWORD = "password"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table1(time,device_id,s1) values(1, 'd1', 1)", + String.format(CREATE_USER_FORMAT, USER_1, PASSWORD), + "GRANT SELECT ON TABLE table1 TO USER " + USER_1, + String.format(CREATE_USER_FORMAT, USER_2, PASSWORD) + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void maintainAuthTest() { + // case 1: explain + // user1 with select on table1 + String[] expectedHeader = new String[] {"distribution plan"}; + tableQueryNoVerifyResultTest( + "explain select * from test.table1", expectedHeader, USER_1, PASSWORD); + // user2 without select on table1 + tableAssertTestFail( + "explain select * from test.table1", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", + USER_2, + PASSWORD); + + // case 2: explain analyze [verbose] + // user1 with select on table1 + expectedHeader = new String[] {"Explain Analyze"}; + tableQueryNoVerifyResultTest( + "explain analyze select * from test.table1", expectedHeader, USER_1, PASSWORD); + tableQueryNoVerifyResultTest( + "explain analyze verbose select * from test.table1", expectedHeader, USER_1, PASSWORD); + // user2 without select on table1 + tableAssertTestFail( + "explain analyze select * from test.table1", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", + USER_2, + PASSWORD); + tableAssertTestFail( + "explain analyze verbose select * from test.table1", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", + USER_2, + PASSWORD); + + // case 3: show current_sql_dialect + expectedHeader = new String[] {"CurrentSqlDialect"}; + tableQueryNoVerifyResultTest("SHOW CURRENT_SQL_DIALECT", expectedHeader, USER_2, PASSWORD); + + // case 4: show current_user + expectedHeader = new String[] {"CurrentUser"}; + tableQueryNoVerifyResultTest("SHOW CURRENT_USER", expectedHeader, USER_2, PASSWORD); + + // case 5: show version + expectedHeader = new String[] {"Version", "BuildInfo"}; + tableQueryNoVerifyResultTest("SHOW VERSION", expectedHeader, USER_2, PASSWORD); + + // case 6: show current_timestamp + expectedHeader = new String[] {"CurrentTimestamp"}; + tableQueryNoVerifyResultTest("SHOW CURRENT_TIMESTAMP", expectedHeader, USER_2, PASSWORD); + + // case 7: show variables + // user2 + tableAssertTestFail( + "SHOW VARIABLES", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 8: show cluster_id + // user2 + tableAssertTestFail( + "SHOW CLUSTER_ID", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 9: flush + // user2 + tableAssertTestFail( + "FLUSH", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 10: clear cache + // user2 + tableAssertTestFail( + "CLEAR CACHE", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 11: set configuration + // user2 + tableAssertTestFail( + "SET CONFIGURATION query_timeout_threshold='100000'", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 12: show queries + // non-root users can access its own queries + expectedHeader = + new String[] {"query_id", "start_time", "datanode_id", "elapsed_time", "statement", "user"}; + tableQueryNoVerifyResultTest("show queries", expectedHeader, USER_2, PASSWORD); + + // case 13: kill query + // user2 + tableAssertTestFail( + "kill query '20250206_093300_00001_1'", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 14: load configuration + // user2 + tableAssertTestFail( + "LOAD CONFIGURATION", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 15: set system status + // user2 + tableAssertTestFail( + "SET SYSTEM TO RUNNING", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 16: start repair data + // user2 + tableAssertTestFail( + "START REPAIR DATA", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 17: stop repair data + // user2 + tableAssertTestFail( + "STOP REPAIR DATA", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, only root user is allowed", + USER_2, + PASSWORD); + + // case 18: create function + // user1 + tableAssertTestFail( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_1, + PASSWORD); + // user2 + tableAssertTestFail( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_2, + PASSWORD); + + // case 19: show functions + // user1 + expectedHeader = new String[] {"FunctionName", "FunctionType", "ClassName(UDF)", "State"}; + tableQueryNoVerifyResultTest("SHOW FUNCTIONS", expectedHeader, USER_1, PASSWORD); + // user2 + tableQueryNoVerifyResultTest("SHOW FUNCTIONS", expectedHeader, USER_2, PASSWORD); + + // case 20: create function + // user1 + tableAssertTestFail( + "drop function udsf", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_1, + PASSWORD); + // user2 + tableAssertTestFail( + "drop function udsf", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_2, + PASSWORD); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java new file mode 100644 index 0000000000000..2459d0c81061c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java @@ -0,0 +1,550 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; + +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetFuzzyTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullIdQueryIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE testNullId(id1 STRING TAG, id2 STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD, s3 DOUBLE FIELD)", + "INSERT INTO testNullId(time,id1,id2,s1,s2,s3) " + "values(1, null, null, 0, false, 11.1)", + "CREATE TABLE table1(device_id STRING TAG, color STRING ATTRIBUTE, s1 INT32 FIELD, s2 BOOLEAN FIELD, s3 INT64 FIELD)", + // in seq disk + "INSERT INTO table1(time,device_id,color,s1,s2,s3) " + + "values(1, 'd1', 'green', 1, false, 11)", + "INSERT INTO table1(time,device_id,s1) " + "values(5, 'd1', 5)", + "FLUSH", + // in uneq disk + "INSERT INTO table1(time,device_id,s1,s2,s3) " + "values(4, 'd1', 4, true, 44)", + "INSERT INTO table1(time,device_id,color,s1) " + "values(3, 'd1', 'green', 3)", + "FLUSH", + // in seq memtable + "INSERT INTO table1(time,device_id,s1,s2,s3) " + "values(7, 'd1', 7, false, 77)", + "INSERT INTO table1(time,device_id,s1) " + "values(6, 'd1', 6)", + // in unseq memtable + "INSERT INTO table1(time,device_id,color,s1) " + "values(2, 'd1', 'green', 2)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullFilterTest() throws Exception { + String result = defaultFormatDataTime(1) + ",0,false,11.1"; + try (final Connection connectionIsNull = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connectionIsNull.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + ResultSet resultSet = statement.executeQuery("select * from testNullId where id1 is null"); + assertTrue(resultSet.next()); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + resultSet = statement.executeQuery("select * from testNullId where id1 is not null"); + assertFalse(resultSet.next()); + + resultSet = statement.executeQuery("select * from testNullId where id1 like '%'"); + assertFalse(resultSet.next()); + + resultSet = + statement.executeQuery("select * from testNullId where id1 is null and id2 is null"); + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // The second time we read from cache + resultSet = + statement.executeQuery("select * from testNullId where id1 is null and id2 is null"); + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test deduplication + resultSet = + statement.executeQuery("select * from testNullId where id1 is null or id2 is null"); + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test constant select item + resultSet = statement.executeQuery("select *, 1 from testNullId"); + result = defaultFormatDataTime(1) + ",null,null,0,false,11.1,1"; + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("id1") + + "," + + resultSet.getString("id2") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3") + + "," + + resultSet.getString("_col6"); + + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test boolean between + resultSet = + statement.executeQuery("select * from testNullId where s2 between false and true"); + result = defaultFormatDataTime(1) + ",null,null,0,false,11.1"; + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("id1") + + "," + + resultSet.getString("id2") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test boolean not between + resultSet = + statement.executeQuery("select * from testNullId where s2 not between false and true"); + assertFalse(resultSet.next()); + + // Test same column name + resultSet = statement.executeQuery("select time, s1 as a, s2 as a from testNullId"); + result = defaultFormatDataTime(1) + ",0,false"; + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + assertEquals(3, resultSetMetaData.getColumnCount()); + assertEquals("time", resultSetMetaData.getColumnName(1)); + assertEquals(Types.TIMESTAMP, resultSetMetaData.getColumnType(1)); + assertEquals("a", resultSetMetaData.getColumnName(2)); + assertEquals(Types.INTEGER, resultSetMetaData.getColumnType(2)); + assertEquals("a", resultSetMetaData.getColumnName(3)); + assertEquals(Types.BOOLEAN, resultSetMetaData.getColumnType(3)); + + assertTrue(resultSet.next()); + ans = resultSet.getString(1) + "," + resultSet.getString(2) + "," + resultSet.getString(3); + + assertEquals(result, ans); + assertFalse(resultSet.next()); + } + } + + @Test + public void nullSelectTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where time > 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where time < 7", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where time > 1 and time < 7 and s1 >= 3 and s1 <= 5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where s2 is NULL", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where s2 IS NOT NULL OR s3 IS NOT NULL", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void noMeasurementColumnsSelectTest() { + String[] expectedHeader = new String[] {"time"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,", + "1970-01-01T00:00:00.002Z,", + "1970-01-01T00:00:00.003Z,", + "1970-01-01T00:00:00.004Z,", + "1970-01-01T00:00:00.005Z,", + "1970-01-01T00:00:00.006Z,", + "1970-01-01T00:00:00.007Z," + }; + tableResultSetEqualTest("select time from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"device_id"}; + retArray = new String[] {"d1,", "d1,", "d1,", "d1,", "d1,", "d1,"}; + tableResultSetEqualTest( + "select device_id from table1 where time > 1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"color"}; + retArray = new String[] {"green,", "green,", "green,", "green,", "green,", "green,"}; + tableResultSetEqualTest( + "select color from table1 where time > 1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,", + "1970-01-01T00:00:00.002Z,d1,", + "1970-01-01T00:00:00.003Z,d1,", + "1970-01-01T00:00:00.004Z,d1,", + "1970-01-01T00:00:00.005Z,d1,", + "1970-01-01T00:00:00.006Z,d1,", + }; + tableResultSetEqualTest( + "select time, device_id from table1 where time < 7", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "color"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,green,", + "1970-01-01T00:00:00.002Z,d1,green,", + "1970-01-01T00:00:00.003Z,d1,green,", + "1970-01-01T00:00:00.004Z,d1,green,", + "1970-01-01T00:00:00.005Z,d1,green,", + "1970-01-01T00:00:00.006Z,d1,green,", + }; + tableResultSetEqualTest( + "select time, device_id, color from table1 where time < 7", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color"}; + retArray = + new String[] { + "d1,green,", + }; + tableResultSetEqualTest( + "select device_id, color from table1 where device_id='d1' limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"color"}; + retArray = + new String[] { + "green,", + }; + tableResultSetEqualTest( + "select color from table1 where device_id='d1' limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,", "1970-01-01T00:00:00.004Z,", "1970-01-01T00:00:00.005Z,", + }; + tableResultSetEqualTest( + "select time from table1 where time >= 3 and time <= 5 and device_id='d1'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,", "1970-01-01T00:00:00.004Z,", "1970-01-01T00:00:00.005Z,", + }; + tableResultSetEqualTest( + "select time from table1 where time >= 3 and time < 6 and color='green'", + expectedHeader, + retArray, + DATABASE_NAME); + } + + public void limitOffsetTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 OFFSET 2 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 limit 1 OFFSET 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 OFFSET 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", "1970-01-01T00:00:00.002Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 limit 2", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void showStatementTest() { + String[] expectedHeader = new String[] {"CurrentSqlDialect"}; + String[] retArray = + new String[] { + "TABLE,", + }; + tableResultSetEqualTest("show current_sql_dialect", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"CurrentUser"}; + retArray = + new String[] { + "root,", + }; + tableResultSetEqualTest("show current_user", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"CurrentDatabase"}; + retArray = + new String[] { + DATABASE_NAME + ",", + }; + tableResultSetEqualTest("show current_database", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"Version", "BuildInfo"}; + tableResultSetFuzzyTest("show version", expectedHeader, 1, DATABASE_NAME); + + expectedHeader = new String[] {"Variable", "Value"}; + tableResultSetFuzzyTest("show variables", expectedHeader, 15, DATABASE_NAME); + + expectedHeader = new String[] {"ClusterId"}; + tableResultSetFuzzyTest("show clusterid", expectedHeader, 1, DATABASE_NAME); + + expectedHeader = new String[] {"ClusterId"}; + tableResultSetFuzzyTest("show cluster_id", expectedHeader, 1, DATABASE_NAME); + + expectedHeader = new String[] {"CurrentTimestamp"}; + tableResultSetFuzzyTest("show current_timestamp", expectedHeader, 1, DATABASE_NAME); + } + + @Test + public void setSqlDialectTest() throws SQLException { + createUser("tempuser", "temppw"); + + try (Connection userCon = EnvFactory.getEnv().getConnection("tempuser", "temppw"); + Statement userStmt = userCon.createStatement()) { + assertCurrentSqlDialect(true, userStmt); + + // set Tree to Table + userStmt.execute("set sql_dialect=table"); + assertCurrentSqlDialect(false, userStmt); + + // set Table to Tree + userStmt.execute("set sql_dialect=tree"); + assertCurrentSqlDialect(true, userStmt); + } + } + + @Test + public void setSqlDialectContextCleanTest() throws SQLException { + try (Connection userCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement userStmt = userCon.createStatement()) { + userStmt.execute("create database test1"); + userStmt.execute("use test1"); + userStmt.execute("set sql_dialect=tree"); + assertCurrentSqlDialect(true, userStmt); + userStmt.execute("insert into root.db(time,s1) values (0,1), (1, 3), (2,5)"); + } + } + + public static void assertCurrentSqlDialect(boolean expectedTree, Statement statement) + throws SQLException { + ResultSet resultSet = statement.executeQuery("show current_sql_dialect"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + assertEquals("CurrentSqlDialect", resultSetMetaData.getColumnName(1)); + int count = 0; + while (resultSet.next()) { + assertEquals(expectedTree ? "TREE" : "TABLE", resultSet.getString(1)); + count++; + } + assertEquals(1, count); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java new file mode 100644 index 0000000000000..531d88101ba6a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullValueIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table table1(id1 tag, s1 string)", + "insert into table1 values(0, 'd1', null), (1,'d1', 1)", + "flush", + "insert into table1 values(0, 'd1', 0)", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullTest() { + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = new String[] {"time", "id1", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,", + }; + tableResultSetEqualTest("select * from table1", expectedHeader, retArray, DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java new file mode 100644 index 0000000000000..4557a66e7d14c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBQueryAuthIT { + + private static final String DATABASE_NAME = "test"; + private static final String CREATE_USER_FORMAT = "create user %s '%s'"; + private static final String USER_1 = "user1"; + private static final String USER_2 = "user2"; + private static final String USER_3 = "user3"; + private static final String USER_4 = "user4"; + private static final String USER_5 = "user5"; + private static final String USER_6 = "user6"; + private static final String USER_7 = "user7"; + private static final String USER_8 = "user8"; + private static final String USER_9 = "user9"; + private static final String PASSWORD = "password"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table1(time,device_id,s1) values(1, 'd1', 1)", + "CREATE TABLE table2(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table2(time,device_id,s1) values(1, 'd1', 1)", + String.format(CREATE_USER_FORMAT, USER_1, PASSWORD), + "GRANT SELECT ON ANY TO USER " + USER_1, + String.format(CREATE_USER_FORMAT, USER_2, PASSWORD), + "GRANT SELECT ON DATABASE " + DATABASE_NAME + " TO USER " + USER_2, + String.format(CREATE_USER_FORMAT, USER_3, PASSWORD), + "GRANT SELECT ON TABLE table1 TO USER " + USER_3, + String.format(CREATE_USER_FORMAT, USER_4, PASSWORD), + "GRANT DROP ON TABLE table2 TO USER " + USER_4, + String.format(CREATE_USER_FORMAT, USER_5, PASSWORD), + "GRANT INSERT ON TABLE table2 TO USER " + USER_5, + String.format(CREATE_USER_FORMAT, USER_6, PASSWORD), + "GRANT CREATE ON TABLE table2 TO USER " + USER_6, + String.format(CREATE_USER_FORMAT, USER_7, PASSWORD), + "GRANT ALTER ON TABLE table2 TO USER " + USER_7, + String.format(CREATE_USER_FORMAT, USER_8, PASSWORD), + "GRANT DELETE ON TABLE table2 TO USER " + USER_8, + String.format(CREATE_USER_FORMAT, USER_9, PASSWORD), + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void queryAuthTest() { + // case 1: user1 with SELECT ON ANY + String[] expectedHeader1 = new String[] {"time", "device_id", "s1"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,", + }; + String[] expectedHeader2 = new String[] {"device_id"}; + String[] retArray2 = new String[] {"d1,"}; + String[] expectedHeader3 = new String[] {"count(devices)"}; + String[] retArray3 = new String[] {"1,"}; + + tableResultSetEqualTest( + "select * from table1", expectedHeader1, retArray1, USER_1, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "show devices from table1", expectedHeader2, retArray2, USER_1, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "count devices from table1", expectedHeader3, retArray3, USER_1, PASSWORD, DATABASE_NAME); + + // case 2: user2 with SELECT ON database + tableResultSetEqualTest( + "select * from table1", expectedHeader1, retArray1, USER_2, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "show devices from table1", expectedHeader2, retArray2, USER_2, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "count devices from table1", expectedHeader3, retArray3, USER_2, PASSWORD, DATABASE_NAME); + + // case 3: user3 with SELECT ON table1 + tableResultSetEqualTest( + "select * from table1", expectedHeader1, retArray1, USER_3, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "show devices from table1", expectedHeader2, retArray2, USER_3, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "count devices from table1", expectedHeader3, retArray3, USER_3, PASSWORD, DATABASE_NAME); + + // case 4: user3 with SELECT ON table1, without SELECT ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + + // case 5: user4 with only DROP ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_4, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_4, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_4, + PASSWORD, + DATABASE_NAME); + + // case 6: user5 with only INSERT ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_5, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_5, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_5, + PASSWORD, + DATABASE_NAME); + + // case 7: user6 with only CREATE ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_6, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_6, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_6, + PASSWORD, + DATABASE_NAME); + + // case 8: user7 with only ALTER ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_7, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_7, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_7, + PASSWORD, + DATABASE_NAME); + + // case 9: user8 with only DELETE ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_8, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_8, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_8, + PASSWORD, + DATABASE_NAME); + + // case 10: user9 with nothing + tableAssertTestFail( + "select * from test.table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_9, + PASSWORD); + tableAssertTestFail( + "show devices from test.table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_9, + PASSWORD); + tableAssertTestFail( + "count devices from test.table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_9, + PASSWORD); + + // case 11: user1 with SELECT ON ANY + String[] expectedHeader4 = new String[] {"time", "device_id", "table1_s1", "table2_s1"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,1,", + }; + + tableResultSetEqualTest( + "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", + expectedHeader4, + retArray4, + USER_1, + PASSWORD, + DATABASE_NAME); + + // case 12: user2 with SELECT ON database + tableResultSetEqualTest( + "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", + expectedHeader4, + retArray4, + USER_2, + PASSWORD, + DATABASE_NAME); + + // case 3: user3 with SELECT ON just table1 + tableAssertTestFail( + "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationFunctionDistinctIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationFunctionDistinctIT.java new file mode 100644 index 0000000000000..8f9b5c1eac4c2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationFunctionDistinctIT.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +/** + * In this Class, we construct the scenario using DistinctAccumulator for + * AggregationFunctionDistinct. Other cases are covered by {@link IoTDBTableAggregationIT}. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableAggregationFunctionDistinctIT { + + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table1(time,device_id,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'d01',30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,device_id,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'d01',35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,device_id,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'d01',40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,device_id,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'d01',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'d01',30,35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void countDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = new String[] {"d01,2,2,3,1,2,2,2,1,4,1,"}; + + tableResultSetEqualTest( + "select device_id, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void countIfDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = new String[] {"d01,0,1,1,1,1,1,1,1,1,1,"}; + tableResultSetEqualTest( + "select device_id, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void avgDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,35.0,42500.0,35.0,35.0,", + }; + tableResultSetEqualTest( + "select device_id, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sumDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = new String[] {"d01,70.0,85000.0,105.0,35.0,"}; + tableResultSetEqualTest( + "select device_id, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + String[] retArray = + new String[] { + "d01,d01,30,35000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,", + }; + + tableResultSetEqualTest( + "select device_id, min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + String[] retArray = + new String[] { + "d01,d01,40,50000,40.0,35.0,true,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe30,2024-09-24T06:15:50.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:35.000Z,", + }; + + tableResultSetEqualTest( + "select device_id, max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z," + }; + tableResultSetEqualTest( + "select device_id, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,40,50000,40.0,35.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe30,2024-09-24T06:15:50.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:35.000Z," + }; + tableResultSetEqualTest( + "select device_id, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extremeDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = new String[] {"d01,40,50000,40.0,35.0,"}; + tableResultSetEqualTest( + "select device_id, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void varianceDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = new String[] {"d01,25.0,5.625E7,16.7,0.0,"}; + tableResultSetEqualTest( + "select device_id, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java new file mode 100644 index 0000000000000..dc42960be3173 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -0,0 +1,5271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.buildHeaders; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.repeatTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableAggregationIT { + protected static final String DATABASE_NAME = "test"; + protected static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(province STRING TAG, city STRING TAG, region STRING TAG, device_id STRING TAG, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d05','red','A',30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','pudong','d05','red','A',35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d05','red','A',40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d05','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d05','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d09','red','A',30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','chaoyang','d09','red','A',35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d09','red','A',40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d09','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d09','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d13','red','A',30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','haidian','d13','red','A',35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d13','red','A',40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d13','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d13','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','haidian','d15','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d15','yellow','A',36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','haidian','d15','yellow','A',41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','haidian','d15','yellow','A',46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','haidian','d15','yellow','A',51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ================================================================== + // ==================== Normal Aggregation Test ===================== + // ================================================================== + @Test + public void countTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select count(*) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count('a') from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count(*) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count(1) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(*) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,0,0,1,0,1,0,1,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(s1), count(s2), count(s3), count(s4), count(s5), count(s6), count(s7), count(s8), count(s9), count(s10) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,chaoyang,d10,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,chaoyang,d11,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,chaoyang,d12,1,1,1,1,1,1,1,1,3,1,", + "beijing,beijing,haidian,d13,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,haidian,d14,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,haidian,d15,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,haidian,d16,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,huangpu,d01,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,huangpu,d02,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,huangpu,d03,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,huangpu,d04,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,pudong,d05,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,pudong,d06,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,pudong,d07,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,pudong,d08,1,1,1,1,1,1,1,1,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, count(s1), count(s2), count(s3), count(s4), count(s5), count(s6), count(s7), count(s8), count(s9), count(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,5,", + "beijing,beijing,chaoyang,d10,3,", + "beijing,beijing,chaoyang,d11,5,", + "beijing,beijing,chaoyang,d12,3,", + "beijing,beijing,haidian,d13,5,", + "beijing,beijing,haidian,d14,3,", + "beijing,beijing,haidian,d15,5,", + "beijing,beijing,haidian,d16,3,", + "shanghai,shanghai,huangpu,d01,5,", + "shanghai,shanghai,huangpu,d02,3,", + "shanghai,shanghai,huangpu,d03,5,", + "shanghai,shanghai,huangpu,d04,3,", + "shanghai,shanghai,pudong,d05,5,", + "shanghai,shanghai,pudong,d06,3,", + "shanghai,shanghai,pudong,d07,5,", + "shanghai,shanghai,pudong,d08,3,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,count(*) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,device_id,count(1) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,16,", + "beijing,beijing,haidian,16,", + "shanghai,shanghai,huangpu,16,", + "shanghai,shanghai,pudong,16,", + }; + tableResultSetEqualTest( + "select province,city,region,count(*) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,count(1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,32,", "shanghai,shanghai,32,", + }; + tableResultSetEqualTest( + "select province,city,count(*) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,count(1) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,32,", "shanghai,32,", + }; + tableResultSetEqualTest( + "select province,count(*) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,count(1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest("select count(*) from table1", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest("select count(1) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void countIfTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select count_if(device_id = 'd01') from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest( + "select count_if(true) from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d02,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d02,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d02,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d03,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d03,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d03,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d03,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d03,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d04,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d04,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d04,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d05,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d05,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d05,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d05,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d05,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d06,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d06,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d06,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d07,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d07,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d07,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d07,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d07,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d08,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d08,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d08,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d09,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d09,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d09,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d09,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d09,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d10,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d10,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d10,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d11,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d11,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d11,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d11,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d11,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d12,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d12,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d12,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d13,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d13,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d13,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d13,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d13,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d14,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d14,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d14,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d15,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d15,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d15,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d15,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d15,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d16,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d16,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,count_if(color != 'red') from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,0,0,0,0,0,1,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,1,1,1,0,1,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,0,0,0,0,1,1,1,1,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,0,0,0,0,0,1,0,0,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count_if(s1 is null), count_if(s2 < 50000), count_if(s3 > 30), count_if(s4 < 55), count_if(s5), count_if(s6 like '%pudong%'), count_if(s7 = 'shanghai_pudong_red_B_d06_36'), count_if(s8 is null), count(s9 is null), count_if(s10 is not null) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2,1,2,1,1,0,0,3,5,2,", + "beijing,beijing,chaoyang,d10,1,0,0,1,1,0,0,2,3,1,", + "beijing,beijing,chaoyang,d11,3,2,2,2,0,0,0,3,5,1,", + "beijing,beijing,chaoyang,d12,2,1,0,0,1,0,0,2,3,1,", + "beijing,beijing,haidian,d13,2,1,2,1,1,0,0,3,5,2,", + "beijing,beijing,haidian,d14,1,0,0,1,1,0,0,2,3,1,", + "beijing,beijing,haidian,d15,3,2,2,2,0,0,0,3,5,1,", + "beijing,beijing,haidian,d16,2,1,0,0,1,0,0,2,3,1,", + "shanghai,shanghai,huangpu,d01,2,1,2,1,1,0,0,3,5,2,", + "shanghai,shanghai,huangpu,d02,1,0,0,1,1,0,0,2,3,1,", + "shanghai,shanghai,huangpu,d03,3,2,2,2,0,0,0,3,5,1,", + "shanghai,shanghai,huangpu,d04,2,1,0,0,1,0,0,2,3,1,", + "shanghai,shanghai,pudong,d05,2,1,2,1,1,2,0,3,5,2,", + "shanghai,shanghai,pudong,d06,1,0,0,1,1,1,1,2,3,1,", + "shanghai,shanghai,pudong,d07,3,2,2,2,0,2,0,3,5,1,", + "shanghai,shanghai,pudong,d08,2,1,0,0,1,1,0,2,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, count_if(s1 is null), count_if(s2 < 50000), count_if(s3 > 30), count_if(s4 < 55), count_if(s5), count_if(s6 like '%pudong%'), count_if(s7 = 'shanghai_pudong_red_B_d06_36'), count_if(s8 is null), count(s9 is null), count_if(s10 is not null) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,", + "beijing,beijing,haidian,5,", + "shanghai,shanghai,huangpu,5,", + "shanghai,shanghai,pudong,5,", + }; + tableResultSetEqualTest( + "select province,city,region,count(s3 > 30 and s4 < 55) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,6,", "shanghai,6,", + }; + tableResultSetEqualTest( + "select province,count_if(s5) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "select count_if(device_id) from table1", + "701: Aggregate functions [count_if] should only have one boolean expression as argument", + DATABASE_NAME); + + tableAssertTestFail( + "select count_if(s5, device_id != 'd01') from table1", + "701: Aggregate functions [count_if] should only have one boolean expression as argument", + DATABASE_NAME); + } + + @Test + public void avgTest() { + String[] expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + String[] retArray = + new String[] { + "d01,red,A,45.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + retArray = + new String[] { + "d01,red,A,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, avg(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, avg(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,42500.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,38500.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,42500.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,38500.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,42500.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,38500.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,42500.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,38500.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, avg(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,44.5,", + "beijing,beijing,haidian,44.5,", + "shanghai,shanghai,huangpu,44.5,", + "shanghai,shanghai,pudong,44.5,", + }; + tableResultSetEqualTest( + "select province,city,region,avg(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,44.5,", "shanghai,shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,city,avg(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,44.5,", "shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,avg(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"44.5,"}; + tableResultSetEqualTest("select avg(s4) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void sumTest() { + String[] expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + String[] retArray = + new String[] { + "d01,red,A,90.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, sum(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + retArray = + new String[] { + "d01,red,A,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, sum(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, sum(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, sum(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,85000.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,77000.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,85000.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,77000.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,85000.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,77000.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,85000.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,77000.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, sum(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,267.0,", + "beijing,beijing,haidian,267.0,", + "shanghai,shanghai,huangpu,267.0,", + "shanghai,shanghai,pudong,267.0,", + }; + tableResultSetEqualTest( + "select province,city,region,sum(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,534.0,", "shanghai,shanghai,534.0,", + }; + tableResultSetEqualTest( + "select province,city,sum(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,534.0,", "shanghai,534.0,", + }; + tableResultSetEqualTest( + "select province,sum(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"908.0,"}; + tableResultSetEqualTest("select sum(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void minTest() { + String[] expectedHeader = + new String[] {"device_id", "color", "type", "_col3", "_col4", "_col5", "_col6"}; + String[] retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:30.000Z,35.0,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select device_id, color, type, min(time),min(s4), min(s9), min(s10) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3", "_col4"}; + retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:40.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, min(time),min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by color,type,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30.0,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, min(time),min(s3) from table1 where device_id = 'd01' group by 1, 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,min(time),min(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,min(time),min(s1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,35000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,35000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,35000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,35000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,40000,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,min(time),min(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:30.000Z,35.0,", + "beijing,beijing,haidian,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,pudong,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,city,region,min(time),min(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2", "_col3"}; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,city,min(time),min(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1", "_col2"}; + retArray = + new String[] { + "beijing,2024-09-24T06:15:30.000Z,35.0,", "shanghai,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,min(time),min(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"2024-09-24T06:15:30.000Z,30.0,"}; + tableResultSetEqualTest( + "select min(time),min(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void minByTest() { + String[] expectedHeader = new String[] {"device_id", "color", "type", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:35.000Z,35.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, min_by(time, s4), min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3", "_col4"}; + retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, min_by(time, s4), min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,30.0,", + "d01,2024-09-24T06:15:35.000Z,35.0,", + "d01,2024-09-24T06:15:40.000Z,40.0,", + "d01,null,null,", + "d01,null,null,", + }; + + tableResultSetEqualTest( + "select device_id, min_by(time, s3), min(s3) from table1 where device_id = 'd01' group by date_bin(5s, time), 1 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),min_by(time, s4), min(s4) from table1 group by 1,2,3,4,date_bin(5s, time) order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),min_by(time, s1), min(s1) from table1 group by date_bin(5s, time),1,2,3,4 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,40000,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,min_by(time, s2), min(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"40,"}; + tableResultSetEqualTest( + "select min_by(s1, s10) from table1 where s1=40", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxTest() { + String[] expectedHeader = + new String[] {"device_id", "color", "type", "_col3", "_col4", "_col5", "_col6"}; + String[] retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:55.000Z,55.0,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select device_id,color,type,max(time),max(s4),max(s9),max(s10) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3", "_col4"}; + retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id,color,type,max(time),max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by color,type,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30.0,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, max(time),max(s3) from table1 where device_id = 'd01' group by 1, 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,40.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,46.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,40.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,46.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,40.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,46.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,40.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,46.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,max(time),max(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2", "_col3"}; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,max(time),max(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1", "_col2"}; + retArray = + new String[] { + "beijing,2024-09-24T06:15:55.000Z,55.0,", "shanghai,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,max(time),max(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"2024-09-24T06:15:55.000Z,51.0,"}; + tableResultSetEqualTest( + "select max(time),max(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxByTest() { + String[] expectedHeader = new String[] {"device_id", "color", "type", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, max_by(time, s4), max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3", "_col4"}; + retArray = + new String[] { + "d01,red,A,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, color, type, max_by(time, s4), max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id, color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "d01,2024-09-24T06:15:50.000Z,null,null,", + "d01,2024-09-24T06:15:55.000Z,null,null,", + }; + + tableResultSetEqualTest( + "select device_id, date_bin(5s, time), max_by(time, s3), max(s3) from table1 where device_id = 'd01' group by date_bin(5s, time), 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),max_by(time, s4), max(s4) from table1 group by 1,2,3,4,date_bin(5s, time) order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),max_by(time, s1), max(s1) from table1 group by date_bin(5s, time),1,2,3,4 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,40000,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,max_by(time, s2), max(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"40,"}; + tableResultSetEqualTest( + "select max_by(s1, s10) from table1 where s1=40", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void firstTest() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "device_id", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_chaoyang_red_B_d10_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_chaoyang_yellow_A_d11_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_chaoyang_yellow_A_d11_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_chaoyang_yellow_A_d11_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_chaoyang_yellow_A_d11_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_haidian_red_A_d13_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_haidian_red_A_d13_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_haidian_red_B_d14_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_haidian_yellow_A_d15_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_haidian_yellow_A_d15_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_haidian_yellow_A_d15_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_haidian_yellow_A_d15_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_haidian_yellow_B_d16_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_huangpu_red_B_d02_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_huangpu_yellow_A_d03_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_huangpu_yellow_A_d03_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_huangpu_yellow_A_d03_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_pudong_red_A_d05_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_pudong_red_A_d05_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_pudong_red_B_d06_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_pudong_yellow_A_d07_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_pudong_yellow_A_d07_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_pudong_yellow_A_d07_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_pudong_yellow_B_d08_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_pudong_yellow_B_d08_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,beijing_haidian_yellow_A_d15_41,beijing_haidian_yellow_A_d15_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,shanghai_pudong_yellow_A_d07_41,shanghai_pudong_yellow_A_d07_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:35.000Z,35000,", + }; + tableResultSetEqualTest( + "select first_by(time,s2),first(s2) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = + new String[] { + "null,2024-09-24T06:15:30.000Z,", + }; + tableResultSetEqualTest( + "select first_by(s2,time),first(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,null,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,null,null,", + "2024-09-24T06:15:50.000Z,d01,null,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, first_by(time, s4), first(s4) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,null,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first_by(time,s3), first(s3) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,beijing_chaoyang_red_A_d09_35,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,beijing_chaoyang_red_B_d10_36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,beijing_chaoyang_yellow_A_d11_36,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,beijing_chaoyang_yellow_B_d12_30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,beijing_haidian_red_A_d13_35,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,beijing_haidian_red_B_d14_36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,beijing_haidian_yellow_A_d15_36,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,beijing_haidian_yellow_B_d16_30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,shanghai_huangpu_red_A_d01_35,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,shanghai_huangpu_red_B_d02_36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,shanghai_huangpu_yellow_A_d03_36,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,shanghai_huangpu_yellow_B_d04_30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,shanghai_pudong_red_A_d05_35,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,shanghai_pudong_red_B_d06_36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,shanghai_pudong_yellow_A_d07_36,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,shanghai_pudong_yellow_B_d08_30,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,first_by(time,s7),first(s7) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"city", "region", "device_id", "_col3"}; + retArray = + new String[] { + "beijing,chaoyang,d09,null,", + "beijing,chaoyang,d10,true,", + "beijing,chaoyang,d11,null,", + "beijing,chaoyang,d12,true,", + "beijing,haidian,d13,null,", + "beijing,haidian,d14,true,", + "beijing,haidian,d15,null,", + "beijing,haidian,d16,true,", + "shanghai,huangpu,d01,null,", + "shanghai,huangpu,d02,true,", + "shanghai,huangpu,d03,null,", + "shanghai,huangpu,d04,true,", + "shanghai,pudong,d05,null,", + "shanghai,pudong,d06,true,", + "shanghai,pudong,d07,null,", + "shanghai,pudong,d08,true,", + }; + tableResultSetEqualTest( + "select city,region,device_id,first_by(s5,time,time) from table1 group by city,region,device_id order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastTest() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "device_id", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_chaoyang_red_B_d10_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_chaoyang_yellow_A_d11_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_chaoyang_yellow_A_d11_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_chaoyang_yellow_A_d11_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_chaoyang_yellow_A_d11_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_haidian_red_A_d13_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_haidian_red_A_d13_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_haidian_red_B_d14_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_haidian_yellow_A_d15_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_haidian_yellow_A_d15_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_haidian_yellow_A_d15_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_haidian_yellow_A_d15_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_haidian_yellow_B_d16_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_huangpu_red_B_d02_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_huangpu_yellow_A_d03_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_huangpu_yellow_A_d03_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_huangpu_yellow_A_d03_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_pudong_red_A_d05_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_pudong_red_A_d05_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_pudong_red_B_d06_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_pudong_yellow_A_d07_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_pudong_yellow_A_d07_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_pudong_yellow_A_d07_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_pudong_yellow_B_d08_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_pudong_yellow_B_d08_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,beijing_haidian_yellow_A_d15_51,beijing_haidian_yellow_A_d15_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,shanghai_pudong_yellow_A_d07_51,shanghai_pudong_yellow_A_d07_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:50.000Z,50000,", + }; + repeatTest( + "select last_by(time,s2),last(s2) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = + new String[] { + "null,2024-09-24T06:15:55.000Z,", + }; + repeatTest( + "select last_by(s2, time),last(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,null,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,null,null,", + "2024-09-24T06:15:50.000Z,d01,null,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55.0,", + }; + repeatTest( + "select date_bin(5s, time), device_id, last_by(time, s4), last(s4) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,null,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first_by(time,s3), first(s3) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,0xcafebabe55,", + }; + repeatTest( + "select province,city,region,device_id,last_by(time,s8),last(s8) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"city", "region", "device_id", "_col3"}; + retArray = + new String[] { + "beijing,chaoyang,d09,null,", + "beijing,chaoyang,d10,null,", + "beijing,chaoyang,d11,null,", + "beijing,chaoyang,d12,null,", + "beijing,haidian,d13,null,", + "beijing,haidian,d14,null,", + "beijing,haidian,d15,null,", + "beijing,haidian,d16,null,", + "shanghai,huangpu,d01,null,", + "shanghai,huangpu,d02,null,", + "shanghai,huangpu,d03,null,", + "shanghai,huangpu,d04,null,", + "shanghai,pudong,d05,null,", + "shanghai,pudong,d06,null,", + "shanghai,pudong,d07,null,", + "shanghai,pudong,d08,null,", + }; + repeatTest( + "select city,region,device_id,last_by(s5,time) from table1 group by city,region,device_id order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + } + + @Test + public void extremeTest() { + String[] expectedHeader = + new String[] {"device_id", "color", "type", "_col3", "_col4", "_col5"}; + String[] retArray = new String[] {"d01,red,A,55,50000,55.0,"}; + + tableResultSetEqualTest( + "select device_id, color, type, extreme(s1), extreme(s2), extreme(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id,color,type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "color", "type", "_col3"}; + retArray = new String[] {"d01,red,A,40.0,"}; + tableResultSetEqualTest( + "select device_id, color, type, extreme(s3) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id,color, type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id,extreme(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,extreme(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,extreme(s1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,50000,", + "beijing,beijing,chaoyang,d10,50000,", + "beijing,beijing,chaoyang,d11,46000,", + "beijing,beijing,chaoyang,d12,40000,", + "beijing,beijing,haidian,d13,50000,", + "beijing,beijing,haidian,d14,50000,", + "beijing,beijing,haidian,d15,46000,", + "beijing,beijing,haidian,d16,40000,", + "shanghai,shanghai,huangpu,d01,50000,", + "shanghai,shanghai,huangpu,d02,50000,", + "shanghai,shanghai,huangpu,d03,46000,", + "shanghai,shanghai,huangpu,d04,40000,", + "shanghai,shanghai,pudong,d05,50000,", + "shanghai,shanghai,pudong,d06,50000,", + "shanghai,shanghai,pudong,d07,46000,", + "shanghai,shanghai,pudong,d08,40000,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,extreme(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,55.0,", + "beijing,beijing,haidian,55.0,", + "shanghai,shanghai,huangpu,55.0,", + "shanghai,shanghai,pudong,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,extreme(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,55.0,", "shanghai,shanghai,55.0,", + }; + tableResultSetEqualTest( + "select province,city,extreme(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,55.0,", "shanghai,55.0,", + }; + tableResultSetEqualTest( + "select province,extreme(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "51.0,", + }; + tableResultSetEqualTest( + "select extreme(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void groupByAttributeTest() { + + String[] expectedHeader = new String[] {"color", "_col1"}; + String[] retArray = + new String[] { + "red,32,", "yellow,32,", + }; + tableResultSetEqualTest( + "select color, count(*) from table1 group by color order by color", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"type", "_col1"}; + retArray = + new String[] { + "A,40,", "BBBBBBBBBBBBBBBB,24,", + }; + tableResultSetEqualTest( + "select type, count(*) from table1 group by 1 order by type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"color", "type", "_col2"}; + retArray = + new String[] { + "red,A,20,", "red,BBBBBBBBBBBBBBBB,12,", "yellow,A,20,", "yellow,BBBBBBBBBBBBBBBB,12,", + }; + + tableResultSetEqualTest( + "select color,type, count(*) from table1 group by color,type order by color,type", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"color", "type", "_col2", "_col3"}; + retArray = + new String[] { + "red,A,2024-09-24T06:15:30.000Z,4,", + "red,A,2024-09-24T06:15:35.000Z,4,", + "red,A,2024-09-24T06:15:40.000Z,4,", + "red,A,2024-09-24T06:15:50.000Z,4,", + "red,A,2024-09-24T06:15:55.000Z,4,", + "red,BBBBBBBBBBBBBBBB,2024-09-24T06:15:35.000Z,4,", + "red,BBBBBBBBBBBBBBBB,2024-09-24T06:15:40.000Z,4,", + "red,BBBBBBBBBBBBBBBB,2024-09-24T06:15:50.000Z,4,", + "yellow,A,2024-09-24T06:15:30.000Z,4,", + "yellow,A,2024-09-24T06:15:35.000Z,4,", + "yellow,A,2024-09-24T06:15:40.000Z,4,", + "yellow,A,2024-09-24T06:15:45.000Z,4,", + "yellow,A,2024-09-24T06:15:50.000Z,4,", + "yellow,BBBBBBBBBBBBBBBB,2024-09-24T06:15:30.000Z,4,", + "yellow,BBBBBBBBBBBBBBBB,2024-09-24T06:15:40.000Z,4,", + "yellow,BBBBBBBBBBBBBBBB,2024-09-24T06:15:55.000Z,4,", + }; + + tableResultSetEqualTest( + "select color,type, date_bin(5s, time), count(*) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select color,type, date_bin(5s, time), count(1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void groupByValueTest() { + + String[] expectedHeader = new String[] {"s1", "_col1"}; + String[] retArray = + new String[] { + "30,4,", "36,8,", "40,8,", "41,4,", "55,8,", "null,32,", + }; + tableResultSetEqualTest( + "select s1, count(*) from table1 group by s1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s1", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30,1,", + "beijing,beijing,chaoyang,d09,40,1,", + "beijing,beijing,chaoyang,d09,55,1,", + "beijing,beijing,chaoyang,d09,null,2,", + "beijing,beijing,chaoyang,d10,36,1,", + "beijing,beijing,chaoyang,d10,40,1,", + "beijing,beijing,chaoyang,d10,null,1,", + "beijing,beijing,chaoyang,d11,36,1,", + "beijing,beijing,chaoyang,d11,41,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,30,1,", + "beijing,beijing,haidian,d13,40,1,", + "beijing,beijing,haidian,d13,55,1,", + "beijing,beijing,haidian,d13,null,2,", + "beijing,beijing,haidian,d14,36,1,", + "beijing,beijing,haidian,d14,40,1,", + "beijing,beijing,haidian,d14,null,1,", + "beijing,beijing,haidian,d15,36,1,", + "beijing,beijing,haidian,d15,41,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,30,1,", + "shanghai,shanghai,huangpu,d01,40,1,", + "shanghai,shanghai,huangpu,d01,55,1,", + "shanghai,shanghai,huangpu,d01,null,2,", + "shanghai,shanghai,huangpu,d02,36,1,", + "shanghai,shanghai,huangpu,d02,40,1,", + "shanghai,shanghai,huangpu,d02,null,1,", + "shanghai,shanghai,huangpu,d03,36,1,", + "shanghai,shanghai,huangpu,d03,41,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,30,1,", + "shanghai,shanghai,pudong,d05,40,1,", + "shanghai,shanghai,pudong,d05,55,1,", + "shanghai,shanghai,pudong,d05,null,2,", + "shanghai,shanghai,pudong,d06,36,1,", + "shanghai,shanghai,pudong,d06,40,1,", + "shanghai,shanghai,pudong,d06,null,1,", + "shanghai,shanghai,pudong,d07,36,1,", + "shanghai,shanghai,pudong,d07,41,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s1,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s2", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,35000,1,", + "beijing,beijing,chaoyang,d09,50000,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,50000,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,31000,1,", + "beijing,beijing,chaoyang,d11,46000,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,40000,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,35000,1,", + "beijing,beijing,haidian,d13,50000,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,50000,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,31000,1,", + "beijing,beijing,haidian,d15,46000,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,40000,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,35000,1,", + "shanghai,shanghai,huangpu,d01,50000,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,50000,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,31000,1,", + "shanghai,shanghai,huangpu,d03,46000,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,40000,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,35000,1,", + "shanghai,shanghai,pudong,d05,50000,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,50000,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,31000,1,", + "shanghai,shanghai,pudong,d07,46000,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,40000,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s2,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s3", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30.0,1,", + "beijing,beijing,chaoyang,d09,35.0,1,", + "beijing,beijing,chaoyang,d09,40.0,1,", + "beijing,beijing,chaoyang,d09,null,2,", + "beijing,beijing,chaoyang,d10,null,3,", + "beijing,beijing,chaoyang,d11,41.0,1,", + "beijing,beijing,chaoyang,d11,51.0,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,30.0,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,30.0,1,", + "beijing,beijing,haidian,d13,35.0,1,", + "beijing,beijing,haidian,d13,40.0,1,", + "beijing,beijing,haidian,d13,null,2,", + "beijing,beijing,haidian,d14,null,3,", + "beijing,beijing,haidian,d15,41.0,1,", + "beijing,beijing,haidian,d15,51.0,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,30.0,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,30.0,1,", + "shanghai,shanghai,huangpu,d01,35.0,1,", + "shanghai,shanghai,huangpu,d01,40.0,1,", + "shanghai,shanghai,huangpu,d01,null,2,", + "shanghai,shanghai,huangpu,d02,null,3,", + "shanghai,shanghai,huangpu,d03,41.0,1,", + "shanghai,shanghai,huangpu,d03,51.0,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,30.0,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,30.0,1,", + "shanghai,shanghai,pudong,d05,35.0,1,", + "shanghai,shanghai,pudong,d05,40.0,1,", + "shanghai,shanghai,pudong,d05,null,2,", + "shanghai,shanghai,pudong,d06,null,3,", + "shanghai,shanghai,pudong,d07,41.0,1,", + "shanghai,shanghai,pudong,d07,51.0,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,30.0,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s3,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,35.0,1,", + "beijing,beijing,chaoyang,d09,55.0,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,40.0,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,36.0,1,", + "beijing,beijing,chaoyang,d11,46.0,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,55.0,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,35.0,1,", + "beijing,beijing,haidian,d13,55.0,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,40.0,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,36.0,1,", + "beijing,beijing,haidian,d15,46.0,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,55.0,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,35.0,1,", + "shanghai,shanghai,huangpu,d01,55.0,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,40.0,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,36.0,1,", + "shanghai,shanghai,huangpu,d03,46.0,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,55.0,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,35.0,1,", + "shanghai,shanghai,pudong,d05,55.0,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,40.0,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,36.0,1,", + "shanghai,shanghai,pudong,d07,46.0,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,55.0,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s4,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s5", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,false,1,", + "beijing,beijing,chaoyang,d09,true,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,true,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,false,1,", + "beijing,beijing,chaoyang,d11,null,4,", + "beijing,beijing,chaoyang,d12,true,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,false,1,", + "beijing,beijing,haidian,d13,true,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,true,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,false,1,", + "beijing,beijing,haidian,d15,null,4,", + "beijing,beijing,haidian,d16,true,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,false,1,", + "shanghai,shanghai,huangpu,d01,true,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,true,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,false,1,", + "shanghai,shanghai,huangpu,d03,null,4,", + "shanghai,shanghai,huangpu,d04,true,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,false,1,", + "shanghai,shanghai,pudong,d05,true,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,true,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,false,1,", + "shanghai,shanghai,pudong,d07,null,4,", + "shanghai,shanghai,pudong,d08,true,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s5,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s6", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_30,1,", + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_35,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_36,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_41,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_51,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,beijing_chaoyang_yellow_B_d12_55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_30,1,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_35,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_36,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_41,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_51,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,beijing_haidian_yellow_B_d16_55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_30,1,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_35,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_36,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_41,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_51,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,shanghai_huangpu_yellow_B_d04_55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_30,1,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_35,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_36,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_41,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_51,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,shanghai_pudong_yellow_B_d08_55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s6,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s7", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_35,1,", + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_40,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_36,1,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_40,1,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_50,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_36,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_46,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,beijing_chaoyang_yellow_B_d12_30,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_35,1,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_40,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_36,1,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_40,1,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_50,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_36,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_46,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,beijing_haidian_yellow_B_d16_30,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_35,1,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_40,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_36,1,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_40,1,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_50,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_36,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_46,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,shanghai_huangpu_yellow_B_d04_30,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_35,1,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_40,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_36,1,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_40,1,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_50,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_36,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_46,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,shanghai_pudong_yellow_B_d08_30,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s7,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,device_id,s7,count(1) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s8", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,0xcafebabe30,1,", + "beijing,beijing,chaoyang,d09,0xcafebabe55,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,0xcafebabe50,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,0xcafebabe31,1,", + "beijing,beijing,chaoyang,d11,0xcafebabe41,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,0xcafebabe55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,0xcafebabe30,1,", + "beijing,beijing,haidian,d13,0xcafebabe55,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,0xcafebabe50,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,0xcafebabe31,1,", + "beijing,beijing,haidian,d15,0xcafebabe41,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,0xcafebabe55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,0xcafebabe30,1,", + "shanghai,shanghai,huangpu,d01,0xcafebabe55,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,0xcafebabe50,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,0xcafebabe31,1,", + "shanghai,shanghai,huangpu,d03,0xcafebabe41,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,0xcafebabe55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,0xcafebabe30,1,", + "shanghai,shanghai,pudong,d05,0xcafebabe55,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,0xcafebabe50,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,0xcafebabe31,1,", + "shanghai,shanghai,pudong,d07,0xcafebabe41,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,0xcafebabe55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s8,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s9", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s9,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s10", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24,2,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,2024-09-24,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,2024-09-24,1,", + "beijing,beijing,chaoyang,d11,null,4,", + "beijing,beijing,chaoyang,d12,2024-09-24,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,2024-09-24,2,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,2024-09-24,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,2024-09-24,1,", + "beijing,beijing,haidian,d15,null,4,", + "beijing,beijing,haidian,d16,2024-09-24,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,2024-09-24,2,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,2024-09-24,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,2024-09-24,1,", + "shanghai,shanghai,huangpu,d03,null,4,", + "shanghai,shanghai,huangpu,d04,2024-09-24,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,2024-09-24,2,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,2024-09-24,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,2024-09-24,1,", + "shanghai,shanghai,pudong,d07,null,4,", + "shanghai,shanghai,pudong,d08,2024-09-24,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s10,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastQueryTest() { + + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id='d01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d09,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d12,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id,date_bin(5s,time),last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by province,city,region,device_id,date_bin(5s,time) order by device_id,date_bin(5s,time)", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s,time),last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd08', 'd12', 'd13') group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd05', 'd08', 'd09', 'd12', 'd13') group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "beijing,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd05', 'd08', 'd09', 'd12', 'd13') group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d03,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "d05,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d07,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id, last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where city = 'shanghai' and type='A' group by province,city,region,device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void subQueryTest() { + + String[] expectedHeader = new String[] {"ts", "type", "color", "device_id", "current_s7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:50.000Z,BBBBBBBBBBBBBBBB,red,d02,shanghai_huangpu_red_B_d02_50,", + "2024-09-24T06:15:50.000Z,BBBBBBBBBBBBBBBB,red,d06,shanghai_pudong_red_B_d06_50,", + }; + + tableResultSetEqualTest( + "SELECT ts, type, color, device_id, current_s7 FROM (SELECT type, color, device_id, last(time) as ts, last_by(s7,time) as current_s7 FROM table1 WHERE city='shanghai' GROUP BY type, color, device_id) WHERE strpos(current_s7, color) != 0 order by type, color, device_id, ts", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"color", "device_id"}; + retArray = + new String[] { + "red,d01,", + "red,d05,", + "red,d09,", + "red,d13,", + "yellow,d03,", + "yellow,d07,", + "yellow,d11,", + "yellow,d15,", + }; + tableResultSetEqualTest( + "SELECT color, device_id FROM (SELECT date_bin(5s, time), color, device_id, avg(s4) as avg_s4 FROM table1 WHERE type='A' AND (time >= 2024-09-24T06:15:30.000+00:00 AND time <= 2024-09-24T06:15:59.999+00:00) GROUP BY 1,2,3) WHERE avg_s4 > 1.0 GROUP BY color, device_id HAVING count(*) >= 2 ORDER BY color, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT color, device_id FROM (SELECT date_bin(5s, time), color, device_id, avg(s4) as avg_s4 FROM table1 WHERE type='A' AND (time >= 2024-09-24T06:15:30.000+00:00 AND time <= 2024-09-24T06:15:59.999+00:00) GROUP BY 1,2,3) WHERE avg_s4 > 1.0 GROUP BY color, device_id HAVING count(1) >= 2 ORDER BY color, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "city", "type", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,A,2.0,", + "2024-09-24T06:15:40.000Z,beijing,A,2.0,", + "2024-09-24T06:15:50.000Z,beijing,A,1.0,", + "2024-09-24T06:15:30.000Z,beijing,BBBBBBBBBBBBBBBB,1.0,", + "2024-09-24T06:15:40.000Z,beijing,BBBBBBBBBBBBBBBB,1.0,", + "2024-09-24T06:15:50.000Z,beijing,BBBBBBBBBBBBBBBB,1.0,", + "2024-09-24T06:15:30.000Z,shanghai,A,2.0,", + "2024-09-24T06:15:40.000Z,shanghai,A,2.0,", + "2024-09-24T06:15:50.000Z,shanghai,A,1.0,", + "2024-09-24T06:15:30.000Z,shanghai,BBBBBBBBBBBBBBBB,1.0,", + "2024-09-24T06:15:40.000Z,shanghai,BBBBBBBBBBBBBBBB,1.0,", + "2024-09-24T06:15:50.000Z,shanghai,BBBBBBBBBBBBBBBB,1.0,", + }; + + tableResultSetEqualTest( + "SELECT date_bin(10s, five_seconds), city, type, sum(five_seconds_count) / 2 FROM (SELECT date_bin(5s, time) AS five_seconds, city, type, count(*) AS five_seconds_count FROM table1 WHERE (time >= 2024-09-24T06:15:30.000+00:00 AND time <= 2024-09-24T06:15:59.999+00:00) AND device_id IS NOT NULL GROUP BY 1, city, type, device_id HAVING avg(s1) > 1) GROUP BY 1, city, type order by 2,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT date_bin(10s, five_seconds), city, type, sum(five_seconds_count) / 2 FROM (SELECT date_bin(5s, time) AS five_seconds, city, type, count(1) AS five_seconds_count FROM table1 WHERE (time >= 2024-09-24T06:15:30.000+00:00 AND time <= 2024-09-24T06:15:59.999+00:00) AND device_id IS NOT NULL GROUP BY 1, city, type, device_id HAVING avg(s1) > 1) GROUP BY 1, city, type order by 2,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void specialCasesTest() { + String[] expectedHeader = new String[] {"device_id"}; + String[] retArray = + new String[] { + "d01,", "d02,", "d03,", "d04,", "d05,", "d06,", "d07,", "d08,", "d09,", "d10,", "d11,", + "d12,", "d13,", "d14,", "d15,", "d16,", + }; + tableResultSetEqualTest( + "SELECT device_id FROM table1 GROUP BY device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,", + "2024-09-24T06:15:31.000Z,", + "2024-09-24T06:15:35.000Z,", + "2024-09-24T06:15:36.000Z,", + "2024-09-24T06:15:40.000Z,", + "2024-09-24T06:15:41.000Z,", + "2024-09-24T06:15:46.000Z,", + "2024-09-24T06:15:50.000Z,", + "2024-09-24T06:15:51.000Z,", + "2024-09-24T06:15:55.000Z," + }; + tableResultSetEqualTest( + "select time from table1 group by time order by time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void modeTest() { + // AggTableScan + Agg mixed test + String[] expectedHeader = buildHeaders(11); + String[] retArray = + new String[] { + "A,null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "A,null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + }; + tableResultSetEqualTest( + "select mode(type), mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6),mode(s7),mode(s8),mode(s9),mode(s10) from table1 group by city", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void approxCountDistinctTest() { + String[] expectedHeader = buildHeaders(17); + String[] retArray = new String[] {"10,2,2,4,16,2,2,5,5,5,5,2,24,32,5,10,1,"}; + tableResultSetEqualTest( + "select approx_count_distinct(time), approx_count_distinct(province), approx_count_distinct(city), approx_count_distinct(region), approx_count_distinct(device_id), approx_count_distinct(color), approx_count_distinct(type), approx_count_distinct(s1), approx_count_distinct(s2), approx_count_distinct(s3), approx_count_distinct(s4), approx_count_distinct(s5), approx_count_distinct(s6), approx_count_distinct(s7), approx_count_distinct(s8), approx_count_distinct(s9), approx_count_distinct(s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select approx_count_distinct(time, 0.02), approx_count_distinct(province, 0.02), approx_count_distinct(city, 0.02), approx_count_distinct(region, 0.02), approx_count_distinct(device_id, 0.02), approx_count_distinct(color, 0.02), approx_count_distinct(type, 0.02), approx_count_distinct(s1, 0.02), approx_count_distinct(s2, 0.02), approx_count_distinct(s3, 0.02), approx_count_distinct(s4, 0.02), approx_count_distinct(s5, 0.02), approx_count_distinct(s6, 0.02), approx_count_distinct(s7, 0.02), approx_count_distinct(s8, 0.02), approx_count_distinct(s9, 0.02), approx_count_distinct(s10, 0.02) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,2,2,", + "2024-09-24T06:15:31.000Z,beijing,0,0,", + "2024-09-24T06:15:35.000Z,beijing,2,2,", + "2024-09-24T06:15:36.000Z,beijing,2,4,", + "2024-09-24T06:15:40.000Z,beijing,0,4,", + "2024-09-24T06:15:41.000Z,beijing,2,0,", + "2024-09-24T06:15:46.000Z,beijing,0,2,", + "2024-09-24T06:15:50.000Z,beijing,0,2,", + "2024-09-24T06:15:51.000Z,beijing,2,0,", + "2024-09-24T06:15:55.000Z,beijing,2,0,", + "2024-09-24T06:15:30.000Z,shanghai,2,2,", + "2024-09-24T06:15:31.000Z,shanghai,0,0,", + "2024-09-24T06:15:35.000Z,shanghai,2,2,", + "2024-09-24T06:15:36.000Z,shanghai,2,4,", + "2024-09-24T06:15:40.000Z,shanghai,0,4,", + "2024-09-24T06:15:41.000Z,shanghai,2,0,", + "2024-09-24T06:15:46.000Z,shanghai,0,2,", + "2024-09-24T06:15:50.000Z,shanghai,0,2,", + "2024-09-24T06:15:51.000Z,shanghai,2,0,", + "2024-09-24T06:15:55.000Z,shanghai,2,0,", + }; + + tableResultSetEqualTest( + "select time,province,approx_count_distinct(s6),approx_count_distinct(s7) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2", "_col3"}, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,province,approx_count_distinct(s6,0.02),approx_count_distinct(s7,0.02) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2", "_col3"}, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select approx_count_distinct(time,0.0040625),approx_count_distinct(time,0.26) from table1", + new String[] {"_col0", "_col1"}, + new String[] {"10,11,"}, + DATABASE_NAME); + } + + @Test + public void exceptionTest() { + tableAssertTestFail( + "select avg() from table1", + "701: Aggregate functions [avg] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select sum() from table1", + "701: Aggregate functions [sum] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select extreme() from table1", + "701: Aggregate functions [extreme] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select first() from table1", + "701: Aggregate functions [first] should only have one or two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select first_by() from table1", + "701: Aggregate functions [first_by] should only have two or three arguments", + DATABASE_NAME); + tableAssertTestFail( + "select last() from table1", + "701: Aggregate functions [last] should only have one or two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select last_by() from table1", + "701: Aggregate functions [last_by] should only have two or three arguments", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct() from table1", + "701: Aggregate functions [approx_count_distinct] should only have two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 0.3) from table1", + "750: Max Standard Error must be in [0.0040625, 0.26]: 0.3", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 0.3) from table1", + "750: Max Standard Error must be in [0.0040625, 0.26]: 0.3", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 'test') from table1", + "701: Second argument of Aggregate functions [approx_count_distinct] should be numberic type and do not use expression", + DATABASE_NAME); + } + + // ================================================================== + // ===================== Select Distinct Test ======================= + // ================================================================== + + // Select distinct is a special kind of aggregate query in actual, so we put ITs here to reuse the + // test data. + + @Test + public void simpleTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = new String[] {"30,", "36,", "40,", "41,", "55,", "null,"}; + tableResultSetEqualTest( + "select distinct s1 from table1 order by s1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"region", "s1"}; + retArray = + new String[] { + "chaoyang,30,", + "chaoyang,36,", + "chaoyang,40,", + "chaoyang,41,", + "chaoyang,55,", + "chaoyang,null,", + "haidian,30,", + "haidian,36,", + "haidian,40,", + "haidian,41,", + "haidian,55,", + "haidian,null,", + "huangpu,30,", + "huangpu,36,", + "huangpu,40,", + "huangpu,41,", + "huangpu,55,", + "huangpu,null,", + "pudong,30,", + "pudong,36,", + "pudong,40,", + "pudong,41,", + "pudong,55,", + "pudong,null," + }; + tableResultSetEqualTest( + "select distinct region, s1 from table1 order by region, s1", + expectedHeader, + retArray, + DATABASE_NAME); + + // show all devices + expectedHeader = new String[] {"province", "city", "region", "device_id"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,", + "beijing,beijing,chaoyang,d10,", + "beijing,beijing,chaoyang,d11,", + "beijing,beijing,chaoyang,d12,", + "beijing,beijing,haidian,d13,", + "beijing,beijing,haidian,d14,", + "beijing,beijing,haidian,d15,", + "beijing,beijing,haidian,d16,", + "shanghai,shanghai,huangpu,d01,", + "shanghai,shanghai,huangpu,d02,", + "shanghai,shanghai,huangpu,d03,", + "shanghai,shanghai,huangpu,d04,", + "shanghai,shanghai,pudong,d05,", + "shanghai,shanghai,pudong,d06,", + "shanghai,shanghai,pudong,d07,", + "shanghai,shanghai,pudong,d08,", + }; + tableResultSetEqualTest( + "select distinct province,city,region,device_id from table1 order by province,city,region,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,"}; + tableResultSetEqualTest( + "select distinct s1 < 0 from table1 where s1 is not null", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void withGroupByTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = new String[] {"30,", "36,", "40,", "41,", "55,", "null,"}; + tableResultSetEqualTest( + "select distinct s1 from table1 group by s1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct s1 from table1 group by s1,s2 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"30.0,", "36.0,", "40.0,", "41.0,", "55.0,", "null,"}; + tableResultSetEqualTest( + "select distinct avg(s1) from table1 group by s1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct avg(s1) from table1 group by s1,s2 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = new String[] {"4,", "8,", "32,"}; + tableResultSetEqualTest( + "select distinct count(*) from table1 group by s1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = new String[] {"4,", "8,"}; + tableResultSetEqualTest( + "select distinct count(*) from table1 group by s1, s2 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest1() { + tableAssertTestFail( + "select distinct s1 from table1 order by s2", + "701: For SELECT DISTINCT, ORDER BY expressions must appear in select list", + DATABASE_NAME); + } + + // ================================================================== + // ================== Agg-Function Distinct Test ==================== + // ================================================================== + @Test + public void countDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "2,2,4,16,5,5,5,5,2,24,32,5,10,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select count(distinct province), count(distinct city), count(distinct region), count(distinct device_id), count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,5,5,5,2,6,8,5,10,1,", + "beijing,beijing,haidian,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,huangpu,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,pudong,5,5,5,5,2,6,8,5,10,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,5,5,5,5,2,6,8,5,10,1,", + "haidian,5,5,5,5,2,6,8,5,10,1,", + "huangpu,5,5,5,5,2,6,8,5,10,1,", + "pudong,5,5,5,5,2,6,8,5,10,1," + }; + tableResultSetEqualTest( + "select region, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void countIfDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "1,1,1,1,0,1,1,1,1,1,1,1,1,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select count_if(distinct province = 'shanghai'), count_if(distinct city = 'shanghai'), count_if(distinct region= 'huangpu'), count_if(distinct device_id = 'd03'), count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,0,1,1,1,1,1,1,1,1,1,", + "beijing,beijing,haidian,0,1,1,1,1,1,1,1,1,1,", + "shanghai,shanghai,huangpu,0,1,1,1,1,1,1,1,1,1,", + "shanghai,shanghai,pudong,0,1,1,1,1,1,1,1,1,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,0,1,1,1,1,1,1,1,1,1,", + "haidian,0,1,1,1,1,1,1,1,1,1,", + "huangpu,0,1,1,1,1,1,1,1,1,1,", + "pudong,0,1,1,1,1,1,1,1,1,1," + }; + tableResultSetEqualTest( + "select region, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void avgDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "40.4,40400.0,39.4,42.4,", + }; + // global Aggregation + tableResultSetEqualTest( + "select avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,40.4,40400.0,39.4,42.4,", + "beijing,beijing,haidian,40.4,40400.0,39.4,42.4,", + "shanghai,shanghai,huangpu,40.4,40400.0,39.4,42.4,", + "shanghai,shanghai,pudong,40.4,40400.0,39.4,42.4," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,40.4,40400.0,39.4,42.4,", + "haidian,40.4,40400.0,39.4,42.4,", + "huangpu,40.4,40400.0,39.4,42.4,", + "pudong,40.4,40400.0,39.4,42.4," + }; + tableResultSetEqualTest( + "select region, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sumDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "202.0,202000.0,197.0,212.0,", + }; + // global Aggregation + tableResultSetEqualTest( + "select sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,202.0,202000.0,197.0,212.0,", + "beijing,beijing,haidian,202.0,202000.0,197.0,212.0,", + "shanghai,shanghai,huangpu,202.0,202000.0,197.0,212.0,", + "shanghai,shanghai,pudong,202.0,202000.0,197.0,212.0," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,202.0,202000.0,197.0,212.0,", + "haidian,202.0,202000.0,197.0,212.0,", + "huangpu,202.0,202000.0,197.0,212.0,", + "pudong,202.0,202000.0,197.0,212.0," + }; + tableResultSetEqualTest( + "select region, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "beijing,beijing,chaoyang,d01,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select min(distinct province), min(distinct city), min(distinct region), min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,30,31000,30.0,35.0,false,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,30,31000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,30,31000,30.0,35.0,false,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + retArray = + new String[] { + "chaoyang,d09,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "haidian,d13,30,31000,30.0,35.0,false,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "huangpu,d01,30,31000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "pudong,d05,30,31000,30.0,35.0,false,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + retArray = + new String[] { + "d03,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "d11,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z," + }; + + // test GroupByDistinctAccumulator + tableResultSetEqualTest( + "select device_id, min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "shanghai,shanghai,pudong,d16,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select max(distinct province), max(distinct city), max(distinct region), max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d12,55,50000,51.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,55,50000,51.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,55,50000,51.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + retArray = + new String[] { + "chaoyang,d12,55,50000,51.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "haidian,d16,55,50000,51.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "huangpu,d04,55,50000,51.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "pudong,d08,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + retArray = + new String[] { + "d03,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "d11,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z," + }; + + // test GroupByDistinctAccumulator + tableResultSetEqualTest( + "select device_id, max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "huangpu,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + // global Aggregation + tableResultSetEqualTest( + "select first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "huangpu,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select region, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "huangpu,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + // global Aggregation + tableResultSetEqualTest( + "select last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), last_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "huangpu,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select region, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extremeDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "55,50000,51.0,55.0,", + }; + // global Aggregation + tableResultSetEqualTest( + "select extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,55,50000,51.0,55.0,", + "beijing,beijing,haidian,55,50000,51.0,55.0,", + "shanghai,shanghai,huangpu,55,50000,51.0,55.0,", + "shanghai,shanghai,pudong,55,50000,51.0,55.0," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,55,50000,51.0,55.0,", + "haidian,55,50000,51.0,55.0,", + "huangpu,55,50000,51.0,55.0,", + "pudong,55,50000,51.0,55.0," + }; + tableResultSetEqualTest( + "select region, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void varianceDistinctTest() { + // The addInput logic of all variance functions are the same, so test any one function is ok. + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "68.2,4.824E7,49.0,54.6,", + }; + tableResultSetEqualTest( + "select round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,68.2,4.824E7,49.0,54.6,", + "beijing,beijing,haidian,68.2,4.824E7,49.0,54.6,", + "shanghai,shanghai,huangpu,68.2,4.824E7,49.0,54.6,", + "shanghai,shanghai,pudong,68.2,4.824E7,49.0,54.6," + }; + tableResultSetEqualTest( + "select province,city,region, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,68.2,4.824E7,49.0,54.6,", + "haidian,68.2,4.824E7,49.0,54.6,", + "huangpu,68.2,4.824E7,49.0,54.6,", + "pudong,68.2,4.824E7,49.0,54.6," + }; + tableResultSetEqualTest( + "select region, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void mixedTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "32,5,", + }; + + tableResultSetEqualTest( + "select count(s1), count(distinct s1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,3,", + "beijing,beijing,chaoyang,d10,2,2,", + "beijing,beijing,chaoyang,d11,2,2,", + "beijing,beijing,chaoyang,d12,1,1,", + "beijing,beijing,haidian,d13,3,3,", + "beijing,beijing,haidian,d14,2,2,", + "beijing,beijing,haidian,d15,2,2,", + "beijing,beijing,haidian,d16,1,1,", + "shanghai,shanghai,huangpu,d01,3,3,", + "shanghai,shanghai,huangpu,d02,2,2,", + "shanghai,shanghai,huangpu,d03,2,2,", + "shanghai,shanghai,huangpu,d04,1,1,", + "shanghai,shanghai,pudong,d05,3,3,", + "shanghai,shanghai,pudong,d06,2,2,", + "shanghai,shanghai,pudong,d07,2,2,", + "shanghai,shanghai,pudong,d08,1,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,count(s1), count(distinct s1) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void singleInputDistinctAggregationTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "5,40.4,", + }; + + tableResultSetEqualTest( + "select count(distinct s1), avg(distinct s1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"16,"}; + tableResultSetEqualTest( + "select count(distinct device_id) from table1 group by date_bin(1d,time) order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,40.4,", + "beijing,beijing,haidian,5,40.4,", + "shanghai,shanghai,huangpu,5,40.4,", + "shanghai,shanghai,pudong,5,40.4," + }; + tableResultSetEqualTest( + "select province,city,region,count(distinct s1), avg(distinct s1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest2() { + tableAssertTestFail( + "select count(distinct *) from table1", + "mismatched input '*'. Expecting: ", + DATABASE_NAME); + + String errMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Unsupported expression: Row"; + tableAssertTestFail("select distinct (s1,s2) from table1", errMsg, DATABASE_NAME); + + tableAssertTestFail("select (s1,s2) from table1", errMsg, DATABASE_NAME); + + tableAssertTestFail("select * from table1 where (s1,s2) is not null", errMsg, DATABASE_NAME); + } + + @Test + public void emptyBlockInStreamOperatorTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = new String[] {}; + + // the sub-query produces empty block + + // test StreamingHashAggregationOperator + tableResultSetEqualTest( + "select count(1) from (select * from table1 where s1 + 1 < 1) group by device_id,s1", + expectedHeader, + retArray, + DATABASE_NAME); + + // test StreamingAggregationOperator + tableResultSetEqualTest( + "select count(1) from (select * from table1 where s1 + 1 < 1) group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationNonStreamIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationNonStreamIT.java new file mode 100644 index 0000000000000..5a402b81ff784 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationNonStreamIT.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableAggregationNonStreamIT extends IoTDBTableAggregationIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + String original = createSqls[2]; + // make 'province', 'city', 'region' be FIELD to cover cases using GroupedAccumulator + createSqls[2] = + "CREATE TABLE table1(province STRING FIELD, city STRING FIELD, region STRING FIELD, device_id STRING TAG, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)"; + prepareTableData(createSqls); + // rollback original content + createSqls[2] = original; + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java new file mode 100644 index 0000000000000..0e6e2124f328f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery; + +public class SubqueryDataSetUtils { + public static final String DATABASE_NAME = "subqueryTest"; + public static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; + public static final String[] CREATE_SQLS = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + // table1 + "CREATE TABLE table1(province STRING TAG, city STRING TAG, region STRING TAG, device_id STRING TAG, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',50,50,50.0,50.0,true,'shanghai_huangpu_red_A_d01_50','shanghai_huangpu_red_A_d01_50',X'cafebabe50',2024-09-24T06:15:00.000+00:00,'2024-09-25')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:16:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',60,60,60.0,60.0,false,'shanghai_huangpu_red_A_d01_60','shanghai_huangpu_red_A_d01_60',X'cafebabe60',2024-09-24T06:16:00.000+00:00,'2024-09-26')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:17:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',70,70,70.0,70.0,true,'shanghai_huangpu_red_A_d01_70','shanghai_huangpu_red_A_d01_70',X'cafebabe70',2024-09-24T06:17:00.000+00:00,'2024-09-27')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d05','red','A',30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','pudong','d05','red','A',35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d05','red','A',40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d05','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d05','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d09','red','A',30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','chaoyang','d09','red','A',35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d09','red','A',40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d09','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d09','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d13','red','A',30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','haidian','d13','red','A',35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d13','red','A',40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d13','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d13','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','haidian','d15','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d15','yellow','A',36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','haidian','d15','yellow','A',41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','haidian','d15','yellow','A',46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','haidian','d15','yellow','A',51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + // table2 + "CREATE TABLE table2(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " values(1, 'd1', 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + + " values(2, 'd1', 2, 22, 2.2, 22.2, false)", + "INSERT INTO table2(time,device_id,s6,s7,s8,s9,s10) " + + " values(3, 'd1', 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO table2(time,device_id,s6,s7,s8,s9,s10) " + + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + // table3 + "CREATE TABLE table3(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d_null',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d_null',40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java new file mode 100644 index 0000000000000..495b93d33457c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.correlated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.ONLY_SUPPORT_EQUI_JOIN; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCorrelatedExistsSubqueryIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCorrelatedExistsSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: exists and other filter + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: only exists + sql = "SELECT s1 FROM table3 t3 WHERE exists(SELECT s1 from table1 t1 WHERE t1.s1 = t3.s1)"; + retArray = new String[] {"30,", "30,", "40,"}; + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: exists with distinct + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: aggregation in exists + sql = + "SELECT cast(min(%s) as INT32) as %s FROM table1 t1 WHERE exists(SELECT avg(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: aggregation with group by in exists(subquery returns empty result with having + // clause) + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT count(*) from table3 t3 where t1.%s = t3.%s group by device_id having count(*) > 5)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: limit 1 in exists + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s limit 1)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testNestedExistsSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: Nested exists + sql = + "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25))"; + retArray = new String[] {"30,"}; + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: Nested exists with not(64 - 4 = 60 rows remain) + sql = + "select count(*) as cnt from table1 t1 where not exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25))"; + retArray = new String[] {"60,"}; + expectedHeader = new String[] {"cnt"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testMultipleExistsSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: multiple exists + sql = + "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1) and exists(select s1 from table2 t2 where t2.s1 = t1.s1 - 25)"; + retArray = new String[] {"30,"}; + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: multiple exists with not + sql = + "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1) and not exists(select s1 from table2 t2 where t2.s1 = t1.s1)"; + retArray = new String[] {"30,", "40,"}; + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: with InPredicate + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and %s in (select %s from table3)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with InPredicate in exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s not in (select %s from table2 where %s is not null))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Scalar Subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 > (select min(%s) from table3)"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Scalar Subquery in exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and s1 = (select min(%s) from table3))"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Quantified Comparison + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 != any(select %s from table2)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Quantified Comparison in exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s = any (select %s from table3))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testCorrelatedExistsSubqueryWithMultipleCorrelation() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: Multiple correlation in exists + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and not exists(SELECT (%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testCorrelatedExistsSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: exists in having + sql = + "SELECT device_id, count(*) from table1 t1 group by device_id having exists(SELECT 1 from table3 t3 where t3.device_id = t1.device_id)"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {"d01,5,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + sql = + "SELECT device_id, count(*) from table1 t1 group by device_id having exists(SELECT 1 from table3 t3 where t3.device_id != 'd01' and t3.device_id = t1.device_id)"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: not exists in having + sql = + "SELECT device_id, count(*) from table1 t1 group by device_id having count(*) = 5 and not exists(SELECT 1 from table3 t3 where t3.device_id = t1.device_id)"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {"d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } + + @Test + public void testCorrelatedExistsSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: exists in Select clause + sql = + "select exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; + retArray = new String[] {"true,", "true,"}; + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = "select exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3"; + retArray = new String[] {"true,", "true,", "true,", "false,"}; + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "select not exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; + retArray = new String[] {"false,", "false,"}; + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = "select not exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3"; + retArray = new String[] {"false,", "false,", "false,", "true,"}; + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testNonComparisonFilterInCorrelatedExistsSubquery() { + String errMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": " + ONLY_SUPPORT_EQUI_JOIN; + // Legality check: Correlated subquery with Non-equality comparison is not support for now. + tableAssertTestFail( + "select s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 > t3.s1)", + errMsg, + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 >= t3.s1)", + errMsg, + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 < t3.s1)", + errMsg, + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 <= t3.s1)", + errMsg, + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 != t3.s1)", + errMsg, + DATABASE_NAME); + } + + @Test + public void testCorrelatedExistsLegalityCheck() { + // Legality check: Correlated subqueries can only access columns from the immediately outer + // scope and cannot access columns from the further outer queries. + tableAssertTestFail( + "select s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t1.s1))", + "701: Given correlated subquery is not supported", + DATABASE_NAME); + + // Legality check: Correlated subqueries with limit clause and limit count greater than 1 is not + // supported for now + tableAssertTestFail( + "select s1 from table3 t3 where 30 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 limit 2)", + "Decorrelation for LIMIT with row count greater than 1 is not supported yet", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java new file mode 100644 index 0000000000000..c4c41f2e196d7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.correlated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCorrelatedQuantifiedComparisonIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCorrelatedQuantifiedComparisonInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= any(SELECT %s from table3 t3 where t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= some(SELECT %s from table3 t3 where t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= all(SELECT %s from table3 t3 where t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + sql = + "SELECT %s >= any(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + sql = + "SELECT %s >= some(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + sql = + "SELECT %s >= all(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } + + @Test + public void testCorrelatedQuantifiedComparisonLegalityCheck() { + // Legality check: Correlated subqueries can only access columns from the immediately outer + // scope and cannot access columns from the further outer queries. + tableAssertTestFail( + "select s1 from table1 t1 where s1 >= any(select s1 from table3 t3 where t1.s1 = t3.s1 and s1 >= any(select s1 from table2 t2 where t2.s1 = t1.s1))", + "701: Given correlated subquery is not supported", + DATABASE_NAME); + + // Legality check: Correlated subqueries with limit clause and limit count greater than 1 is not + // supported for now + tableAssertTestFail( + "select s1 from table3 t3 where 30 = t3.s1 and s1 >= any(select s1 from table2 t2 where t2.s1 = t3.s1 limit 2)", + "Decorrelation for LIMIT with row count greater than 1 is not supported yet", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java new file mode 100644 index 0000000000000..ef0921d9f384e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java @@ -0,0 +1,417 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.correlated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCorrelatedScalarSubqueryIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCorrelatedScalarSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: Aggregation with correlated filter in scalar subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT max(%s) from table3 t3 where t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: Non-Aggregation with correlated filter in scalar subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT distinct %s from table3 t3 where t1.%s = t3.%s and %s > 30)"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: limit 1 in scalar subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT %s from table3 t3 where t1.%s = t3.%s and %s > 30 limit 1)"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testNestedCorrelatedScalarSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: Nested exists + sql = + "select distinct s1 from table1 t1 where s1 >= (select max(s1) from table3 t3 where t1.s1 = t3.s1 and s1 = (select max(s1) from table1 t1_2 where t1_2.s1 = t3.s1))"; + retArray = new String[] {"30,", "40,"}; + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testMultipleScalarSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: multiple scalar subquery + sql = + "select distinct s1 from table1 t1 where s1 = (select max(s1) from table3 t3 where t1.s1 = t3.s1) and s1 = (select min(s1) from table3 t3 where t1.s1 = t3.s1)"; + retArray = new String[] {"30,", "40,"}; + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testCorrelatedScalarSubqueryInWhereClauseWithOtherCorrelatedSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + // Test case: with Exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and exists(select s1 from table3 t3 where t1.s1 = t3.s1)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: with In Predicate + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and %s in (select %s from table3)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with InPredicate in scalar subquery + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s not in (select %s from table2 where %s is not null))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Scalar Subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 > (select min(%s) from table3)"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with nested Scalar Subquery + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and s1 = (select min(%s) from table3))"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Quantified Comparison + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 != any(select %s from table2)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Quantified Comparison in scalar subquery + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s = any (select %s from table3))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testCorrelatedScalarSubqueryWithMultipleCorrelation() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: Multiple correlation in exists + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testCorrelatedScalarSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: scalar subquery in having + sql = + "SELECT device_id, count(*) from table1 t1 group by device_id having count(*) + 35 = (SELECT max(s1) from table3 t3 where t3.device_id = t1.device_id)"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {"d01,5,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } + + @Test + public void testCorrelatedScalarSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: exists in Select clause + sql = + "select s1 = (select max(s1) from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; + retArray = new String[] {"true,", "true,"}; + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = "select (select max(s1) from table1 t1 where t1.s1 = t3.s1) from table3 t3"; + retArray = new String[] {"30,", "30,", "40,", "null,"}; + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testNonEqualityComparisonFilterInCorrelatedScalarSubquery() { + // Legality check: Correlated subquery with Non-equality comparison is not support for now. + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 > t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 >= t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 < t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 <= t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 != t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + } + + @Test + public void testCorrelatedScalarSubqueryLegalityCheck() { + // Legality check: Correlated subqueries can only access columns from the immediately outer + // scope and cannot access columns from the further outer queries. + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select s1 from table3 t3 where t1.s1 = t3.s1 and s1 = (select s1 from table2 t2 where t2.s1 = t1.s1 limit 1) limit 1)", + "701: Given correlated subquery is not supported", + DATABASE_NAME); + + // Legality check: Correlated subqueries with limit clause and limit count greater than 1 is not + // supported for now. + tableAssertTestFail( + "select s1 from table3 t3 where 30 = t3.s1 and s1 > (select max(s1) from table2 t2 where t2.s1 = t3.s1 limit 2)", + "701: Given correlated subquery is not supported", + DATABASE_NAME); + + // Legality check: Scalar subquery should return only one row. + tableAssertTestFail( + "select s1 from table1 t1 where s1 >= (select s1 from table3 t3 where t3.s1 = t1.s1)", + "701: Scalar sub-query has returned multiple rows", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java new file mode 100644 index 0000000000000..1f424347e0722 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedExistsSubqueryIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testUnCorrelatedExistsSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: aggregation in exist subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: group by in exist subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 group by device_id)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: expression in select clause with exist subquery + sql = + "SELECT cast(%s AS INT32) + 1 as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 group by device_id)"; + retArray = new String[] {"31,", "41,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: exists with other conjuncts + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and exists(select max(s1) from table1)"; + retArray = new String[] {"30,", "30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: false exist + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and exists(select s1 from table1 where s1 < 0)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: limit in exist subquery + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 limit 10)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: Nested exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 where exists(select s1 from table2))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: Multiple exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select avg(s1) from table1 limit 1) and exists(select s1 from table1 where exists(select s1 from table2))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testUnCorrelatedNotExistsSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: aggregation in exist subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and not exists(select max(s1) from table1)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: limit in exist subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE not exists(select s1 from table1 limit 1)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: subquery has output with not exists + sql = + "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and not exists(select s1 from table1 where s1 < 0)"; + retArray = new String[] {"30,", "30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with distinct + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and not exists(select s1 from table1 where s1 < 0)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: Nested exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 where not exists(select s1 from table1 where s1 < 0))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: Multiple exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and not exists(select s1 from table1 where s1 < 0) and exists(select s1 from table1 where not exists(select s1 from table1 where s1 < 0))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: with InPredicate + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select max(s1) from table1) and s1 in (select s1 from table3)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: with InPredicate in exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 where s1 in (select s1 from table3))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: with Scalar Subquery + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select s1 from table1) and s1 = (select min(s1) from table3)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Scalar Subquery in exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s < 50 and exists(select s1 from table1 where s1 = (select min(s1) from table3))"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Quantified Comparison + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select s1 from table1) and s1 = any(select s1 from table3)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: with Quantified Comparison in exists + sql = + "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = 30 and exists(select s1 from table1 where s1 = any (select s1 from table3))"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testUnCorrelatedExistsSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: exists in having + sql = + "SELECT device_id, count(*) from table1 group by device_id having exists(SELECT %s from table1) and count(*) = 5"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + sql = + "SELECT device_id, count(*) from table1 group by device_id having exists(SELECT %s from table1 where s1 < 0)"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: not exists in having + sql = + "SELECT device_id, count(*) from table1 group by device_id having not exists(SELECT %s from table1 where s1 < 0) and count(*) = 5"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + sql = + "SELECT device_id, count(*) from table1 group by device_id having not exists(SELECT %s from table1) and count(*) = 5"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } + + @Test + public void testUnCorrelatedExistsSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: exists in Select clause + sql = + "SELECT exists(select max(s1) from table1) as %s FROM table3 t3 WHERE %s is not null and device_id = 'd01'"; + retArray = new String[] {"true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + sql = + "SELECT not exists(select max(s1) from table1) as %s FROM table3 t3 WHERE %s is not null and device_id = 'd01'"; + retArray = new String[] {"false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java new file mode 100644 index 0000000000000..ad26fb555909f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedInPredicateSubqueryIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInPredicateSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery returns empty set + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery), subquery returns empty set. Should return all rows + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT min(%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01' and (%s) > (select avg(%s) from table3 where device_id = 'd01'))"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains expression. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s AS INT32) in (SELECT cast(((%s) + 30) AS INT32) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), s contains null value + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {"30,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: where s not in (subquery), s contains null value, the resutl should be empty + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 not in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testInPredicateSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: having s in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 30 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s in (subquery), subquery returns empty set + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd010')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery), subquery returns empty set, should return all rows + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd11') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testInPredicateSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select s in (subquery) + sql = + "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s not in (subquery) + sql = + "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 not in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. 36 not in(30, 40, null) returns + // null + sql = "SELECT s1 not in (SELECT s1 from table3) from table1 where device_id = 'd02'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testInPredicateSubqueryLegalityCheck() { + // Legality check: Multiple parentheses around subquery, this behaves the same as Trino + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 not in ((select s1 from table3 where device_id = 'd01'))", + "701: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: Join key type mismatch.(left key is int and right key is double) + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 in (select s1 + 30.0 from table3 where device_id = 'd01')", + "701: Join key type mismatch", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select s1, s2 in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select (s1, s2) in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 in select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java new file mode 100644 index 0000000000000..11eb1feab9014 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -0,0 +1,674 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedQuantifiedComparisonIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s > any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s > some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s >= any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s >= some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s < any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where < some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s <= any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s <= some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s = any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s = some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s != any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s != some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testAllComparisonInWhereClauseWithoutNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s > all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s >= all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s < all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s <= all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s = all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s != all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testAnyAndSomeComparisonInWhereClauseWithNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s1 > any (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT s1 FROM table3)"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 > some (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT s1 FROM table3)"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 >= any (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT s1 FROM table3)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 >= some (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT s1 FROM table3)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 < any (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT s1 FROM table3)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 < some (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT s1 FROM table3)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testAllComparisonInWhereClauseWithNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s1 > all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 >= all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 < all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 <= all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 = all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 != all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s as INT32) != all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonInWhereWithExpression() { + String sql; + String[] expectedHeader; + String[] retArray; + + sql = + "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 > any (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,", "80,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 > some (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,", "80,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 >= all (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"80,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: having s >= any(subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 >= any(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s >= some(subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 >= some(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s >= all(subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 35 >= all(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + public void testQuantifiedComparisonInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select s > any(subquery) + sql = + "SELECT %s > any(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s > some(subquery) + sql = + "SELECT %s > some(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s > all(subquery) + sql = + "SELECT %s > all(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s < any(subquery), subquery contains null value + sql = "SELECT %s < any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s < some(subquery), subquery contains null value + sql = "SELECT %s < some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s <= any(subquery), subquery contains null value + sql = "SELECT %s <= any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s <= some(subquery), subquery contains null value + sql = "SELECT %s <= some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s != all(subquery), subquery contains null value + sql = "SELECT %s != all(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s != all(subquery), subquery result contains null value and s not in + // non-null + // value result set + sql = + "SELECT %s != all(SELECT (%s) from table3 where device_id = 'd_null') from table1 where device_id = 'd02' and %s != 30"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonLegalityCheck() { + // Legality check: only support any/some/all quantifier + tableAssertTestFail( + "select s1 from table1 where s1 > any_value (select s1 from table3)", + "mismatched input", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java new file mode 100644 index 0000000000000..d2bce2e0d4ae8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedScalarSubqueryIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // region test in where clause + @Test + public void testScalarSubqueryAfterComparisonInOneTable() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: s equals to the maximum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s not equals to the maximum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the average value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= ((SELECT AVG(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the max value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s is less than the maximum value of s in table1 and greater than the minimum value + // of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT max(%s) from table1 WHERE device_id = 'd01') and %s > (SELECT min(%s) from table1 WHERE device_id = 'd01') "; + retArray = new String[] {"40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the avg value of s in table1 and s5 = true + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT avg(%s) FROM table1 WHERE device_id = 'd01' and s5 = true))"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the count value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s less than the sum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT sum(%s) FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: subquery is not aggregation but returns exactly one row + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT %s FROM table1 WHERE device_id = 'd01' and %s = 30)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testScalarSubqueryAfterComparisonInDifferentTables() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: s greater than the count value of s in table2 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) from table2)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s less than the max value of s in table2 * the count value of s in table2 * 10 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < ((SELECT max(%s) from table2) * (SELECT count(%s) from table2)) * 10"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testNestedScalarSubqueryAfterComparison() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: nested scalar subquery in where clause + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: nested scalar subquery with table subquery + sql = + "SELECT %s from (SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')))"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testScalarSubqueryAfterComparisonLegalityCheck() { + // Legality check: subquery returns multiple rows (should fail) + tableAssertTestFail( + "select s1 from table1 where s1 = (select s1 from table1)", + "701: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 = (select s1 from)", "mismatched input", DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 = select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: Main query can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 = (select max(s1) from table1) and", + "mismatched input", + DATABASE_NAME); + } + + // endregion + + // region test in select clause + @Test + public void testScalarSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select scalar subquery as one constant column + sql = + "SELECT cast((SELECT max(%s) from table1 where device_id = 'd01') AS INT32) as %s from table1 where device_id = 'd01'"; + retArray = new String[] {"70,", "70,", "70,", "70,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: scalar subquery in arithmetic expression and function expression + sql = + "SELECT cast((s1 + (SELECT max(%s) from table1 where device_id = 'd01')) AS INT32) as %s from table1 where device_id = 'd01'"; + retArray = new String[] {"100,", "110,", "120,", "130,", "140,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(((SELECT max(%s) from table1 where device_id = 'd01') + (SELECT max(%s) from table1 where device_id = 'd01')) AS INT32) as %s from table1 where device_id = 'd01'"; + retArray = new String[] {"140,", "140,", "140,", "140,", "140,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: scalar subquery in udf expression + sql = + "SELECT cast(floor(sin((SELECT max(%s) from table1 where device_id = 'd01'))) AS INT32) as %s from table1 where device_id = 'd01'"; + retArray = new String[] {"0,", "0,", "0,", "0,", "0,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: scalar subquery in aggregation function + sql = "SELECT count((select count(s1) from table1)) as %s from table1"; + retArray = new String[] {"64,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + // endregion + + // region test in having clause + @Test + public void testScalarSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: scalar subquery in having clause + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) > 3 + (SELECT count(*) from table2 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + // endregion +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterBetweenTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterBetweenTableViewIT.java new file mode 100644 index 0000000000000..345cf0ed58fbb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterBetweenTableViewIT.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFilterBetweenTableViewIT { + protected static final int ITERATION_TIMES = 10; + private static final String DATABASE_NAME = "test"; + private static final List SQLs = new ArrayList<>(); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + createTimeSeries(); + generateTreeData(); + generateTableView(SQLs); + prepareTableData(SQLs); + } + + private static void createTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.vehicle"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s1 with datatype=INT32,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s2 with datatype=INT32,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s3 with datatype=TEXT,encoding=PLAIN"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void generateTreeData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (int i = 1; i <= ITERATION_TIMES; ++i) { + statement.execute( + String.format( + "insert into root.vehicle.d1(timestamp,s1,s2,s3) values(%d,%d,%d,%s)", i, i, i, i)); + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void generateTableView(List SQLs) { + SQLs.add("CREATE DATABASE " + DATABASE_NAME); + SQLs.add("USE " + DATABASE_NAME); + SQLs.add( + "CREATE VIEW table1 (device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 TEXT FIELD) as root.vehicle.**"); + } + + @Test + public void testBetweenExpression() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + int start = 1, end = 5; + String query = "SELECT * FROM table1 WHERE s1 BETWEEN " + start + " AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM table1 WHERE s1 NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = "SELECT * FROM table1 WHERE time BETWEEN " + start + " AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM table1 WHERE time NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = "SELECT * FROM table1 WHERE " + start + " BETWEEN time AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(1), rs.getString("time")); + Assert.assertEquals("1", rs.getString("s1")); + Assert.assertEquals("1", rs.getString("s2")); + Assert.assertEquals("1", rs.getString("s3")); + } + + query = "SELECT * FROM table1 WHERE " + start + " NOT BETWEEN time AND " + end; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start + 1; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = "SELECT * FROM table1 WHERE " + start + " BETWEEN " + end + " AND time"; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertFalse(rs.next()); + } + + query = "SELECT * FROM table1 WHERE " + start + " NOT BETWEEN " + end + " AND time"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(defaultFormatDataTime(i), rs.getString("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = new String[] {"true,true,", "true,false,", "false,false,"}; + tableResultSetEqualTest( + "select s1 between 1 and 2, time between 0 and 1 from table1 where time between 1 and 3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"r1", "r2"}; + retArray = new String[] {"true,true,", "true,false,", "false,false,"}; + tableResultSetEqualTest( + "select s1 between 1 and 2 as r1, time between 0 and 1 as r2 from table1 where time between 1 and 3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"true,true,", "true,false,", "false,false,"}; + tableResultSetEqualTest( + "select s1 between 1 and 2, time between 0 and 1 from table1 where time between 1 and 3 order by device", + expectedHeader, + retArray, + DATABASE_NAME); + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterNullTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterNullTableViewIT.java new file mode 100644 index 0000000000000..1302915e477b2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterNullTableViewIT.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFilterNullTableViewIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE root.testNullFilter", + "CREATE TIMESERIES root.testNullFilter.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.testNullFilter.d1.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", + "CREATE TIMESERIES root.testNullFilter.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN" + }; + + private static final String[] insertSqls = + new String[] { + "INSERT INTO root.testNullFilter.d1(timestamp,s2,s3) " + "values(1, false, 11.1)", + "INSERT INTO root.testNullFilter.d1(timestamp,s1,s2) " + "values(2, 22, true)", + "INSERT INTO root.testNullFilter.d1(timestamp,s1,s3) " + "values(3, 23, 33.3)", + }; + + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW testNullFilter(device STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD, s3 DOUBLE FIELD) as root.testNullFilter.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareData(insertSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullFilterTest() { + String[] retArray = + new String[] { + defaultFormatDataTime(1) + ",null,false,11.1", + defaultFormatDataTime(2) + ",22,true,null", + defaultFormatDataTime(3) + ",23,null,33.3" + }; + try (Connection connectionIsNull = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connectionIsNull.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + int count = 0; + + ResultSet resultSet = statement.executeQuery("select * from testNullFilter where s1 is null"); + while (resultSet.next()) { + String ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(retArray[count], ans); + count++; + } + + resultSet = statement.executeQuery("select * from testNullFilter where s1 is not null"); + while (resultSet.next()) { + String ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(retArray[count], ans); + count++; + } + assertEquals(retArray.length, count); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterTableViewIT.java new file mode 100644 index 0000000000000..1f6bacf097177 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBFilterTableViewIT.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFilterTableViewIT { + protected static final int ITERATION_TIMES = 10; + private static final String DATABASE_NAME = "test"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setUdfMemoryBudgetInMB(5); + EnvFactory.getEnv().initClusterEnvironment(); + createTimeSeries(); + generateData(); + createTableView(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void createTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.vehicle"); + statement.execute( + "create TIMESERIES root.vehicle.testNaN.d1.n1 with datatype=DOUBLE,encoding=PLAIN"); + statement.execute( + "create TIMESERIES root.vehicle.testNaN.d1.n2 with datatype=DOUBLE,encoding=PLAIN"); + statement.execute( + "create TIMESERIES root.vehicle.testTimeSeries.d1.s1 with datatype=BOOLEAN,encoding=PLAIN"); + statement.execute( + "create TIMESERIES root.vehicle.testTimeSeries.d1.s2 with datatype=BOOLEAN,encoding=PLAIN"); + statement.execute( + "create TIMESERIES root.vehicle.testUDTF.d1.s1 with datatype=TEXT,encoding=PLAIN"); + statement.execute( + "create TIMESERIES root.vehicle.testUDTF.d1.s2 with datatype=DOUBLE,encoding=PLAIN"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void generateData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (int i = 0; i < ITERATION_TIMES; i++) { + statement.execute( + String.format( + "insert into root.vehicle.testNaN.d1(timestamp,n1,n2) values(%d,%d,%d)", i, i, i)); + + switch (i % 3) { + case 0: + statement.execute( + String.format( + "insert into root.vehicle.testTimeSeries.d1(timestamp,s1,s2) values(%d,true,true)", + i)); + break; + case 1: + statement.execute( + String.format( + "insert into root.vehicle.testTimeSeries.d1(timestamp,s1,s2) values(%d,true,false)", + i)); + break; + case 2: + statement.execute( + String.format( + "insert into root.vehicle.testTimeSeries.d1(timestamp,s1,s2) values(%d,false,false)", + i)); + break; + } + } + statement.execute( + " insert into root.sg1.d1(time, s1, s2) aligned values (1,1, \"1\"), (2,2,\"2\")"); + statement.execute( + " insert into root.vehicle.testUDTF.d1(time, s1, s2) values (1,\"ss\",0), (2,\"d\",3)"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void createTableView() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW testNaN(device STRING TAG, n1 DOUBLE FIELD, n2 DOUBLE FIELD) as root.vehicle.testNaN.**"); + statement.execute( + "CREATE VIEW testTimeSeries(device STRING TAG, s1 BOOLEAN FIELD, s2 BOOLEAN FIELD) as root.vehicle.testTimeSeries.**"); + statement.execute( + "CREATE VIEW testUDTF(device STRING TAG, s1 TEXT FIELD, s2 DOUBLE FIELD) as root.vehicle.testUDTF.**"); + statement.execute( + "CREATE VIEW sg1(device STRING TAG, s1 DOUBLE FIELD, s2 TEXT FIELD) as root.sg1.**"); + + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testFilterBooleanSeries() { + String[] expectedHeader = new String[] {"s1", "s2"}; + String[] retArray = new String[] {"true,true,", "true,true,", "true,true,", "true,true,"}; + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s2", expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s1 and s2", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "true,true,", + "true,false,", + "true,true,", + "true,false,", + "true,true,", + "true,false,", + "true,true," + }; + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s1 or s2", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select s1, s2 from testTimeSeries " + "Where s1 or false", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testFilterNaN() { + String sqlStr = "select n1 from testNaN where n1/n2 > 0"; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = statement.executeQuery(sqlStr); + int count = 0; + while (resultSet.next()) { + ++count; + } + + // 0.0/0.0 is NaN which should not be kept. + assertEquals(ITERATION_TIMES - 1, count); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testSameConstantWithDifferentType() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + ResultSet resultSet = + statement.executeQuery("select s2 from sg1 where s1 = 1 and s2 >= '1' and s2 <= '2'"); + int count = 0; + while (resultSet.next()) { + ++count; + } + assertEquals(1, count); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCompareWithNull() { + tableResultSetEqualTest( + "select s1 from sg1 where s1 != null", new String[] {"s1"}, new String[] {}, DATABASE_NAME); + tableResultSetEqualTest( + "select s1 from sg1 where s1 <> null", new String[] {"s1"}, new String[] {}, DATABASE_NAME); + tableResultSetEqualTest( + "select s1 from sg1 where s1 = null", new String[] {"s1"}, new String[] {}, DATABASE_NAME); + } + + @Test + public void testCalculateWithNull() { + tableResultSetEqualTest( + "select s1 + null from sg1", + new String[] {"_col0"}, + new String[] {"null,", "null,"}, + DATABASE_NAME); + tableResultSetEqualTest( + "select s1 - null from sg1", + new String[] {"_col0"}, + new String[] {"null,", "null,"}, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBNestedQueryTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBNestedQueryTableViewIT.java new file mode 100644 index 0000000000000..9e97a0b03ed0f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBNestedQueryTableViewIT.java @@ -0,0 +1,505 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNestedQueryTableViewIT { + + private static final String DATABASE_NAME = "db"; + + protected static final int ITERATION_TIMES = 10; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setUdfMemoryBudgetInMB(5); + + EnvFactory.getEnv().initClusterEnvironment(); + createTimeSeries(); + generateData(); + createTableView(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void createTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.vehicle"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s1 with datatype=INT32,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s2 with datatype=INT32,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s3 with datatype=TEXT,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s4 with datatype=STRING,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle.d1.s5 with datatype=DATE,encoding=PLAIN"); + statement.execute( + "CREATE TIMESERIES root.vehicle.d1.s6 with datatype=TIMESTAMP,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.vehicle2.d1.s1 with datatype=FLOAT,encoding=PLAIN"); + statement.execute( + "CREATE TIMESERIES root.vehicle2.d1.s2 with datatype=DOUBLE,encoding=PLAIN"); + statement.execute( + "CREATE TIMESERIES root.vehicle2.d1.empty with datatype=DOUBLE,encoding=PLAIN"); + + statement.execute("CREATE TIMESERIES root.likeTest.d1.s1 with datatype=TEXT,encoding=PLAIN"); + statement.execute( + "CREATE TIMESERIES root.likeTest.d1.s2 with datatype=STRING,encoding=PLAIN"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void generateData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (int i = 1; i <= ITERATION_TIMES; ++i) { + statement.execute( + String.format( + "insert into root.vehicle.d1(timestamp,s1,s2,s3,s4,s6) values(%d,%d,%d,%s,%s,%d)", + i, i, i, i, i, i)); + statement.execute( + (String.format( + "insert into root.vehicle2.d1(timestamp,s1,s2) values(%d,%d,%d)", i, i, i))); + } + statement.execute("insert into root.vehicle.d1(timestamp,s5) values(1,'2024-01-01')"); + statement.execute("insert into root.vehicle.d1(timestamp,s5) values(2,'2024-01-02')"); + statement.execute("insert into root.vehicle.d1(timestamp,s5) values(3,'2024-01-03')"); + + statement.execute( + "insert into root.likeTest.d1(timestamp,s1,s2) values(1,'abcdef', '123456')"); + statement.execute( + "insert into root.likeTest.d1(timestamp,s1,s2) values(2,'_abcdef', '123\\456')"); + statement.execute( + "insert into root.likeTest.d1(timestamp,s1,s2) values(3,'abcdef%', '123#456')"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void createTableView() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "create view vehicle1(device_id STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 TEXT FIELD, s4 STRING FIELD, s5 DATE FIELD, s6 TIMESTAMP FIELD) as root.vehicle.**"); + + statement.execute( + "create view vehicle2(device_id STRING TAG, s1 FLOAT FIELD, s2 DOUBLE FIELD, empty DOUBLE FIELD) as root.vehicle2.**"); + statement.execute( + "create view likeTest(device_id STRING TAG, s1 TEXT FIELD, s2 STRING FIELD) as root.likeTest.**"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Ignore + @Test + public void testNestedRowByRowUDFExpressions() { + String sqlStr = + "select time, s1, s2, sin(sin(s1) * sin(s2) + cos(s1) * cos(s1)) + sin(sin(s1 - s1 + s2) * sin(s2) + cos(s1) * cos(s1)), asin(sin(asin(sin(s1 - s2 / (-s1))))) from vehicle2"; + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + ResultSet resultSet = statement.executeQuery(sqlStr); + + assertEquals(1 + 4, resultSet.getMetaData().getColumnCount()); + + int count = 0; + while (resultSet.next()) { + ++count; + + assertEquals(count, resultSet.getLong(1)); + assertEquals(count, Double.parseDouble(resultSet.getString(2)), 0); + assertEquals(count, Double.parseDouble(resultSet.getString(3)), 0); + assertEquals(2 * Math.sin(1.0), Double.parseDouble(resultSet.getString(4)), 1e-5); + assertEquals( + Math.asin(Math.sin(Math.asin(Math.sin(count - count / (-count))))), + Double.parseDouble(resultSet.getString(5)), + 1e-5); + } + + assertEquals(ITERATION_TIMES, count); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testRawDataQueryWithConstants() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = "SELECT time, 1 + s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i + 1, rs.getInt(2), 0.01); + } + Assert.assertFalse(rs.next()); + } + + query = "SELECT time, (1 + 4) * 2 / 10 + s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i + 1, rs.getInt(2), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testDuplicatedRawDataQueryWithConstants() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = "SELECT time, 1 + s1, 1 + s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i + 1, rs.getInt(2), 0.01); + Assert.assertEquals(i + 1, rs.getInt(3), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testCommutativeLaws() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, s1, s1 + 1, 1 + s1, s1 * 2, 2 * s1 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i, rs.getInt(2)); + Assert.assertEquals(i + 1, rs.getInt(3), 0.01); + Assert.assertEquals(i + 1, rs.getInt(4), 0.01); + Assert.assertEquals(i * 2, rs.getInt(5), 0.01); + Assert.assertEquals(i * 2, rs.getInt(6), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testAssociativeLaws() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, s1, s1 + 1 + 2, (s1 + 1) + 2, s1 + (1 + 2), s1 * 2 * 3, s1 * (2 * 3), (s1 * 2) * 3 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i, rs.getInt(2)); + Assert.assertEquals(i + 3, rs.getInt(3), 0.01); + Assert.assertEquals(i + 3, rs.getInt(4), 0.01); + Assert.assertEquals(i + 3, rs.getInt(5), 0.01); + Assert.assertEquals(i * 6, rs.getInt(6), 0.01); + Assert.assertEquals(i * 6, rs.getInt(7), 0.01); + Assert.assertEquals(i * 6, rs.getInt(8), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testDistributiveLaw() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, s1, (s1 + 1) * 2, s1 * 2 + 1 * 2, (s1 + 1) / 2, s1 / 2 + 1 / 2 FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(i, rs.getInt(2)); + Assert.assertEquals(2 * i + 2, rs.getInt(3), 0.01); + Assert.assertEquals(2 * i + 2, rs.getInt(4), 0.01); + Assert.assertEquals((i + 1) / 2, rs.getInt(5), 0.01); + Assert.assertEquals(i / 2 + 1 / 2, rs.getInt(6), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testOrderOfArithmeticOperations() { + // Priority from high to low: + // 1. exponentiation and root extraction (not supported yet) + // 2. multiplication and division + // 3. addition and subtraction + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + String query = + "SELECT time, 1 + s1 * 2 + 1, (1 + s1) * 2 + 1, (1 + s1) * (2 + 1) FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(2 * i + 2, rs.getInt(2), 0.01); + Assert.assertEquals(2 * i + 3, rs.getInt(3), 0.01); + Assert.assertEquals(3 * i + 3, rs.getInt(4), 0.01); + } + Assert.assertFalse(rs.next()); + } + + query = + "SELECT time, 1 - s1 / 2 + 1, (1 - s1) / 2 + 1, (1 - s1) / (2 + 1) FROM vehicle1 where device_id='d1'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 1; i <= ITERATION_TIMES; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong(1)); + Assert.assertEquals(2 - i / 2, rs.getInt(2), 0.01); + Assert.assertEquals((1 - i) / 2 + 1, rs.getInt(3), 0.01); + Assert.assertEquals((1 - i) / (2 + 1), rs.getInt(4), 0.01); + } + Assert.assertFalse(rs.next()); + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testBetweenExpression() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + int start = 2, end = 8; + statement.execute("USE " + DATABASE_NAME); + String query = + "SELECT * FROM vehicle1 where device_id='d1' and (s1 BETWEEN " + + start + + " AND " + + end + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM vehicle1 where device_id='d1' and (s1 NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(1, rs.getLong("time")); + Assert.assertEquals("1", rs.getString("s1")); + Assert.assertEquals("1", rs.getString("s2")); + Assert.assertEquals("1", rs.getString("s3")); + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM vehicle1 where device_id='d1' and (time BETWEEN " + + start + + " AND " + + end + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + + query = + "SELECT * FROM vehicle1 where device_id='d1' and (time NOT BETWEEN " // test not between + + (end + 1) + + " AND " + + ITERATION_TIMES + + ")"; + try (ResultSet rs = statement.executeQuery(query)) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(1, rs.getLong("time")); + Assert.assertEquals("1", rs.getString("s1")); + Assert.assertEquals("1", rs.getString("s2")); + Assert.assertEquals("1", rs.getString("s3")); + for (int i = start; i <= end; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(i, rs.getLong("time")); + Assert.assertEquals(String.valueOf(i), rs.getString("s1")); + Assert.assertEquals(String.valueOf(i), rs.getString("s2")); + Assert.assertEquals(String.valueOf(i), rs.getString("s3")); + } + } + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testRegularLikeInExpressions() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + String[] ans = new String[] {"abcdef"}; + String query = "SELECT s1 FROM likeTest where s1 LIKE 'abcdef'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"_abcdef"}; + query = "SELECT s1 FROM likeTest where s1 LIKE '\\_%' escape '\\'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"abcdef", "_abcdef", "abcdef%"}; + query = "SELECT s1 FROM likeTest where s1 LIKE '%abcde%' escape '\\'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 5; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"123456"}; + query = "SELECT s2 FROM likeTest where s2 LIKE '12345_'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"123\\456"}; + query = "SELECT s2 FROM likeTest where s2 LIKE '%\\\\%' escape '\\'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + ans = new String[] {"123#456"}; + query = "SELECT s2 FROM likeTest where s2 LIKE '123##456' escape '#'"; + try (ResultSet rs = statement.executeQuery(query)) { + for (int i = 2; i < 3; i++) { + Assert.assertTrue(rs.next()); + Assert.assertEquals(ans[i - 2], rs.getString(1)); + } + Assert.assertFalse(rs.next()); + } + + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java new file mode 100644 index 0000000000000..91f4998ac537b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.apache.tsfile.enums.TSDataType; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSimpleQueryTableViewIT { + private static final String DATABASE_NAME = "test"; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCreateTimeseries1() { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + } catch (SQLException e) { + e.printStackTrace(); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE VIEW table1(device STRING TAG, s1 INT32 FIELD) as root.sg1.**"); + + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + if (resultSet.next() + && resultSet.getString(ColumnHeaderConstant.COLUMN_NAME).equals("s1")) { + assertEquals("INT32", resultSet.getString(ColumnHeaderConstant.DATATYPE).toUpperCase()); + assertEquals( + "FIELD", resultSet.getString(ColumnHeaderConstant.COLUMN_CATEGORY).toUpperCase()); + } + } + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testOrderByTimeDesc() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (1, 1)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (2, 2)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (3, 3)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (4, 4)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s1) VALUES (3, 3)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s1) VALUES (1, 1)"); + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s0 INT32 FIELD, s1 INT32 FIELD) as root.sg1.**"); + statement.execute("flush"); + + String[] expectedHeader = new String[] {"time", "device", "s0", "s1"}; + String[] ret = + new String[] { + defaultFormatDataTime(4) + ",d0,4,null,", + defaultFormatDataTime(3) + ",d0,3,3,", + defaultFormatDataTime(2) + ",d0,2,null,", + defaultFormatDataTime(1) + ",d0,1,1,", + }; + + tableResultSetEqualTest( + "select * from table1 order by time desc", expectedHeader, ret, DATABASE_NAME); + } + } + + @Test + public void testShowTimeseriesDataSet1() throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s2 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s3 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s4 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s5 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s6 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s7 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s8 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s9 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s10 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(5); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD) as root.sg1.**"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(12, count); + + } catch (SQLException e) { + e.printStackTrace(); + throw e; + } + } + + @Test + public void testShowTimeseriesDataSet2() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.setFetchSize(10); + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s2 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s3 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s4 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s5 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s6 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s7 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s8 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s9 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s10 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(10); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD) as root.sg1.**"); + + statement.execute("flush"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(12, count); + + } catch (SQLException e) { + e.printStackTrace(); + throw e; + } + } + + @Test + public void testShowTimeseriesDataSet3() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.setFetchSize(15); + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s2 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s3 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s4 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s5 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s6 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s7 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s8 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s9 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s10 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.setFetchSize(15); + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD, s4 INT32 FIELD, s5 INT32 FIELD, s6 INT32 FIELD, s7 INT32 FIELD, s8 INT32 FIELD, s9 INT32 FIELD, s10 INT32 FIELD) as root.sg1.**"); + + statement.execute("flush"); + + int count = 0; + try (ResultSet resultSet = statement.executeQuery("describe table1")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(12, count); + + } catch (SQLException e) { + e.printStackTrace(); + throw e; + } + } + + @Test + public void testFirstOverlappedPageFiltered() throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + // seq chunk : [1,10] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (1, 1)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (10, 10)"); + + statement.execute("flush"); + + // seq chunk : [13,20] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (13, 13)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (20, 20)"); + + statement.execute("flush"); + + // unseq chunk : [5,15] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (15, 15)"); + + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE VIEW table1(device STRING TAG, s0 INT32 FIELD) as root.sg1.**"); + + long count = 0; + try (ResultSet resultSet = statement.executeQuery("select s0 from table1 where s0 > 18")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(1, count); + } + } + + @Test + public void testPartialInsertion() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + try { + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0, s1) VALUES (1, 1, 2.2)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("s1")); + } + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s0 INT32 FIELD, s1 INT32 FIELD) as root.sg1.**"); + + try (ResultSet resultSet = statement.executeQuery("select s0, s1 from table1")) { + while (resultSet.next()) { + assertEquals(1, resultSet.getInt("s0")); + assertEquals(null, resultSet.getString("s1")); + } + } + } + } + + @Test + public void testOverlappedPagesMerge() throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN"); + + // seq chunk : start-end [1000, 1000] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (1000, 0)"); + + statement.execute("flush"); + + // unseq chunk : [1,10] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (1, 1)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (10, 10)"); + + statement.execute("flush"); + + // usneq chunk : [5,15] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (5, 5)"); + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (15, 15)"); + + statement.execute("flush"); + + // unseq chunk : [15,15] + statement.execute("INSERT INTO root.sg1.d0(timestamp, s0) VALUES (15, 150)"); + + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE VIEW table1(device STRING TAG, s0 INT32 FIELD) as root.sg1.**"); + + long count = 0; + + try (ResultSet resultSet = statement.executeQuery("select s0 from table1 where s0 < 100")) { + while (resultSet.next()) { + count++; + } + } + + Assert.assertEquals(4, count); + } + } + + @Test + public void testTimeseriesMetadataCache() throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.sg1"); + for (int i = 0; i < 10000; i++) { + statement.execute( + "CREATE TIMESERIES root.sg1.d0.s" + i + " WITH DATATYPE=INT32,ENCODING=PLAIN"); + } + for (int i = 1; i < 10000; i++) { + statement.execute("INSERT INTO root.sg1.d0(timestamp, s" + i + ") VALUES (1000, 1)"); + } + statement.execute("flush"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + StringBuilder createTableBuilder = new StringBuilder(); + createTableBuilder.append("CREATE VIEW table1(device STRING TAG,"); + for (int i = 0; i < 1000; i++) { + String columnName = "s" + i; + createTableBuilder.append(columnName).append(" INT32 FIELD,"); + } + createTableBuilder + .deleteCharAt(createTableBuilder.lastIndexOf(",")) + .append(") as root.sg1.**"); + statement.execute(createTableBuilder.toString()); + statement.executeQuery("select s0 from table1"); + } catch (SQLException e) { + fail(); + } + } + + @Test + public void testUseSameStatement() throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.sg1"); + statement.execute( + "CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT64, ENCODING=RLE, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT64, ENCODING=RLE, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s0 WITH DATATYPE=INT64, ENCODING=RLE, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT64, ENCODING=RLE, COMPRESSOR=SNAPPY"); + + statement.execute("insert into root.sg1.d0(timestamp,s0,s1) values(1,1,1)"); + statement.execute("insert into root.sg1.d1(timestamp,s0,s1) values(1000,1000,1000)"); + statement.execute("insert into root.sg1.d0(timestamp,s0,s1) values(10,10,10)"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s0 INT64 FIELD, s1 INT64 FIELD) as root.sg1.**"); + + List resultSetList = new ArrayList<>(); + + ResultSet r1 = statement.executeQuery("select * from table1 where device='d0' and time <= 1"); + resultSetList.add(r1); + + ResultSet r2 = statement.executeQuery("select * from table1 where device='d1' and s0 = 1000"); + resultSetList.add(r2); + + ResultSet r3 = statement.executeQuery("select * from table1 where device='d0' and s1 = 10"); + resultSetList.add(r3); + + r1.next(); + Assert.assertEquals(r1.getLong(1), 1L); + Assert.assertEquals(r1.getLong(3), 1L); + Assert.assertEquals(r1.getLong(4), 1L); + + r2.next(); + Assert.assertEquals(r2.getLong(1), 1000L); + Assert.assertEquals(r2.getLong(3), 1000L); + Assert.assertEquals(r2.getLong(4), 1000L); + + r3.next(); + Assert.assertEquals(r3.getLong(1), 10L); + Assert.assertEquals(r3.getLong(3), 10L); + Assert.assertEquals(r3.getLong(4), 10L); + } + } + + @Test + public void testEnableAlign() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32"); + statement.execute("CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=BOOLEAN"); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD) as root.sg1.**"); + ResultSet resultSet = statement.executeQuery("select time, s1, s2 from table1"); + ResultSetMetaData metaData = resultSet.getMetaData(); + int[] types = {Types.TIMESTAMP, Types.INTEGER, Types.BOOLEAN}; + int columnCount = metaData.getColumnCount(); + for (int i = 0; i < columnCount; i++) { + Assert.assertEquals(types[i], metaData.getColumnType(i + 1)); + } + } + } + + @Test + public void testNewDataType() throws SQLException { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute("CREATE DATABASE root.sg1"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s4 WITH DATATYPE=DATE, ENCODING=PLAIN, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s5 WITH DATATYPE=TIMESTAMP, ENCODING=PLAIN, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s6 WITH DATATYPE=BLOB, ENCODING=PLAIN, COMPRESSOR=SNAPPY"); + statement.execute( + "CREATE TIMESERIES root.sg1.d1.s7 WITH DATATYPE=STRING, ENCODING=PLAIN, COMPRESSOR=SNAPPY"); + for (int i = 1; i <= 10; i++) { + statement.execute( + String.format( + "insert into root.sg1.d1(timestamp, s4, s5, s6, s7) values(%d, \"%s\", %d, %s, \"%s\")", + i, LocalDate.of(2024, 5, i % 31 + 1), i, "X'cafebabe'", i)); + } + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE VIEW table1(device STRING TAG, s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING FIELD) as root.sg1.**"); + + try (ResultSet resultSet = statement.executeQuery("select * from table1")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + final int columnCount = metaData.getColumnCount(); + assertEquals(6, columnCount); + HashMap columnType = new HashMap<>(); + for (int i = 3; i <= columnCount; i++) { + if (metaData.getColumnLabel(i).equals("s4")) { + columnType.put(i, TSDataType.DATE); + } else if (metaData.getColumnLabel(i).equals("s5")) { + columnType.put(i, TSDataType.TIMESTAMP); + } else if (metaData.getColumnLabel(i).equals("s6")) { + columnType.put(i, TSDataType.BLOB); + } else if (metaData.getColumnLabel(i).equals("s7")) { + columnType.put(i, TSDataType.TEXT); + } + } + byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (resultSet.next()) { + long time = resultSet.getLong(1); + Date date = resultSet.getDate(3); + long timestamp = resultSet.getLong(4); + byte[] blob = resultSet.getBytes(5); + String text = resultSet.getString(6); + assertEquals(2024 - 1900, date.getYear()); + assertEquals(5 - 1, date.getMonth()); + assertEquals(time % 31 + 1, date.getDate()); + assertEquals(time, timestamp); + assertArrayEquals(byteArray, blob); + assertEquals(String.valueOf(time), text); + } + } + + } catch (SQLException e) { + fail(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableView2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableView2IT.java new file mode 100644 index 0000000000000..509cddd434ac0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableView2IT.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceTableView2IT extends IoTDBAlignByDeviceTableViewIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableView3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableView3IT.java new file mode 100644 index 0000000000000..573d27d885b06 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableView3IT.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceTableView3IT extends IoTDBAlignByDeviceTableViewIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDegreeOfParallelism(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java new file mode 100644 index 0000000000000..4f33aa49a0d2d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java @@ -0,0 +1,568 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceTableViewIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create view vehicle(device_id STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 STRING FIELD, s4 BOOLEAN FIELD) as root.vehicle.**" + }; + + private static final String[] sqls = + new String[] { + "CREATE DATABASE root.vehicle", + "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE", + "CREATE TIMESERIES root.vehicle.d0.s1 WITH DATATYPE=INT64, ENCODING=RLE", + "CREATE TIMESERIES root.vehicle.d0.s2 WITH DATATYPE=FLOAT, ENCODING=RLE", + "CREATE TIMESERIES root.vehicle.d0.s3 WITH DATATYPE=TEXT, ENCODING=PLAIN", + "CREATE TIMESERIES root.vehicle.d0.s4 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", + "CREATE TIMESERIES root.vehicle.d1.s0 WITH DATATYPE=INT32, ENCODING=RLE", + "insert into root.vehicle.d0(timestamp,s0) values(1,101)", + "insert into root.vehicle.d0(timestamp,s0) values(2,198)", + "insert into root.vehicle.d0(timestamp,s0) values(100,99)", + "insert into root.vehicle.d0(timestamp,s0) values(101,99)", + "insert into root.vehicle.d0(timestamp,s0) values(102,80)", + "insert into root.vehicle.d0(timestamp,s0) values(103,99)", + "insert into root.vehicle.d0(timestamp,s0) values(104,90)", + "insert into root.vehicle.d0(timestamp,s0) values(105,99)", + "insert into root.vehicle.d0(timestamp,s0) values(106,99)", + "insert into root.vehicle.d0(timestamp,s0) values(2,10000)", + "insert into root.vehicle.d0(timestamp,s0) values(50,10000)", + "insert into root.vehicle.d0(timestamp,s0) values(1000,22222)", + "insert into root.vehicle.d0(timestamp,s1) values(1,1101)", + "insert into root.vehicle.d0(timestamp,s1) values(2,198)", + "insert into root.vehicle.d0(timestamp,s1) values(100,199)", + "insert into root.vehicle.d0(timestamp,s1) values(101,199)", + "insert into root.vehicle.d0(timestamp,s1) values(102,180)", + "insert into root.vehicle.d0(timestamp,s1) values(103,199)", + "insert into root.vehicle.d0(timestamp,s1) values(104,190)", + "insert into root.vehicle.d0(timestamp,s1) values(105,199)", + "insert into root.vehicle.d0(timestamp,s1) values(2,40000)", + "insert into root.vehicle.d0(timestamp,s1) values(50,50000)", + "insert into root.vehicle.d0(timestamp,s1) values(1000,55555)", + "insert into root.vehicle.d0(timestamp,s1) values(2000-01-01T00:00:00+08:00, 100)", + "insert into root.vehicle.d0(timestamp,s2) values(1000,55555)", + "insert into root.vehicle.d0(timestamp,s2) values(2,2.22)", + "insert into root.vehicle.d0(timestamp,s2) values(3,3.33)", + "insert into root.vehicle.d0(timestamp,s2) values(4,4.44)", + "insert into root.vehicle.d0(timestamp,s2) values(102,10.00)", + "insert into root.vehicle.d0(timestamp,s2) values(105,11.11)", + "insert into root.vehicle.d0(timestamp,s2) values(1000,1000.11)", + "insert into root.vehicle.d0(timestamp,s3) values(60,'aaaaa')", + "insert into root.vehicle.d0(timestamp,s3) values(70,'bbbbb')", + "insert into root.vehicle.d0(timestamp,s3) values(80,'ccccc')", + "insert into root.vehicle.d0(timestamp,s3) values(101,'ddddd')", + "insert into root.vehicle.d0(timestamp,s3) values(102,'fffff')", + "insert into root.vehicle.d0(timestamp,s3) values(2000-01-01T00:00:00+08:00, 'good')", + "insert into root.vehicle.d0(timestamp,s4) values(100, false)", + "insert into root.vehicle.d0(timestamp,s4) values(100, true)", + "insert into root.vehicle.d1(timestamp,s0) values(1,999)", + "insert into root.vehicle.d1(timestamp,s0) values(1000,888)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : createTableViewSqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,null,null,", + "1970-01-01T00:00:00.060Z,d0,null,null,null,aaaaa,null,", + "1970-01-01T00:00:00.070Z,d0,null,null,null,bbbbb,null,", + "1970-01-01T00:00:00.080Z,d0,null,null,null,ccccc,null,", + "1970-01-01T00:00:00.100Z,d0,99,199,null,null,true,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,ddddd,null,", + "1970-01-01T00:00:00.102Z,d0,80,180,10.0,fffff,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,null,null,", + "1970-01-01T00:00:00.104Z,d0,90,190,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + "1970-01-01T00:00:00.106Z,d0,99,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,null,null,", + "1999-12-31T16:00:00.000Z,d0,null,100,null,good,null,", + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle order by device_id", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void selectTestWithLimitOffset1() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle order by time asc, device_id offset 1 limit 5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectTestWithLimitOffset2() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1999-12-31T16:00:00.000Z,d0,null,100,null,good,null,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,null,null,", + "1970-01-01T00:00:00.106Z,d0,99,null,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle order by device_id desc, time desc offset 1 limit 5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWithDuplicatedPathsTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s0", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,101,1101,", + "1970-01-01T00:00:00.002Z,d0,10000,10000,40000,", + "1970-01-01T00:00:00.050Z,d0,10000,10000,50000,", + "1970-01-01T00:00:00.100Z,d0,99,99,199,", + "1970-01-01T00:00:00.101Z,d0,99,99,199,", + "1970-01-01T00:00:00.102Z,d0,80,80,180,", + "1970-01-01T00:00:00.103Z,d0,99,99,199,", + "1970-01-01T00:00:00.104Z,d0,90,90,190,", + "1970-01-01T00:00:00.105Z,d0,99,99,199,", + "1970-01-01T00:00:00.106Z,d0,99,99,null,", + "1970-01-01T00:00:01.000Z,d0,22222,22222,55555,", + "1999-12-31T16:00:00.000Z,d0,null,null,100,", + "1970-01-01T00:00:00.001Z,d1,999,999,null,", + "1970-01-01T00:00:01.000Z,d1,888,888,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectLimitTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s0", "s1"}; + + String[] retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d0,10000,10000,40000,", + "1970-01-01T00:00:00.050Z,d0,10000,10000,50000,", + "1970-01-01T00:00:00.100Z,d0,99,99,199,", + "1970-01-01T00:00:00.101Z,d0,99,99,199,", + "1970-01-01T00:00:00.102Z,d0,80,80,180,", + "1970-01-01T00:00:00.103Z,d0,99,99,199,", + "1970-01-01T00:00:00.104Z,d0,90,90,190,", + "1970-01-01T00:00:00.105Z,d0,99,99,199,", + "1970-01-01T00:00:00.106Z,d0,99,99,null,", + "1970-01-01T00:00:01.000Z,d0,22222,22222,55555,", + }; + tableResultSetEqualTest( + "select time, device_id, s0,s0,s1 from vehicle order by device_id,time offset 1 limit 10", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWithValueFilterTest() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.100Z,d0,99,199,null,null,true,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,ddddd,null,", + "1970-01-01T00:00:00.102Z,d0,80,180,10.0,fffff,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,null,null,", + "1970-01-01T00:00:00.104Z,d0,90,190,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle where s0 > 0 AND s1 < 200 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectDifferentSeriesWithValueFilterWithoutCacheTest() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.100Z,d0,99,", + "1970-01-01T00:00:00.101Z,d0,99,", + "1970-01-01T00:00:00.102Z,d0,80,", + "1970-01-01T00:00:00.103Z,d0,99,", + "1970-01-01T00:00:00.104Z,d0,90,", + "1970-01-01T00:00:00.105Z,d0,99,", + "1999-12-31T16:00:00.000Z,d0,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s0 from vehicle where s1 < 200 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectDifferentSeriesWithBinaryValueFilterWithoutCacheTest() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.105Z,d0,99,", + }; + tableResultSetEqualTest( + "select time, device_id, s0 from vehicle where s1 < 200 and s2 > 10 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void predicateCannotNormalizedTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,", + "1970-01-01T00:00:00.100Z,d0,99,199,null,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,", + }; + tableResultSetEqualTest( + "select time, device_id, s0,s1,s2 from vehicle where (((\"time\" > 10) AND (\"s1\" > 190)) OR (\"s2\" > 190.0) OR ((\"time\" < 4) AND (\"s1\" > 100))) order by device_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void duplicateProjectionsTest() { + String[] expectedHeader = new String[] {"Time", "device_id", "_col2", "_col3", "alias"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,1102,1102,1102,", + "1970-01-01T00:00:00.002Z,d0,40001,40001,40001,", + "1970-01-01T00:00:00.050Z,d0,50001,50001,50001,", + "1970-01-01T00:00:00.100Z,d0,200,200,200,", + "1970-01-01T00:00:00.101Z,d0,200,200,200,", + "1970-01-01T00:00:00.103Z,d0,200,200,200,", + "1970-01-01T00:00:00.105Z,d0,200,200,200,", + "1970-01-01T00:00:01.000Z,d0,55556,55556,55556,", + }; + tableResultSetEqualTest( + "select Time, device_id, s1+1, s1+1, s1+1 as alias from vehicle where (((\"time\" > 10) AND (\"s1\" > 190)) OR (\"s2\" > 190.0) OR ((\"time\" < 4) AND (\"s1\" > 100))) order by device_id, time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // + // @Test + // public void aggregateTest() { + // String[] retArray = + // new String[] {"root.vehicle.d0,11,11,6,6,1,", "root.vehicle.d1,2,null,null,null,null,"}; + // + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // + // try (ResultSet resultSet = + // statement.executeQuery( + // "select count(s0),count(s1),count(s2),count(s3),count(s4) " + // + "from root.vehicle.d1,root.vehicle.d0 align by device")) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // List actualIndexToExpectedIndexList = + // checkHeader( + // resultSetMetaData, + // "Device,count(s0),count(s1),count(s2),count(s3),count(s4)", + // new int[] { + // Types.VARCHAR, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // }); + // + // int cnt = 0; + // while (resultSet.next()) { + // String[] expectedStrings = retArray[cnt].split(","); + // StringBuilder expectedBuilder = new StringBuilder(); + // StringBuilder actualBuilder = new StringBuilder(); + // for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + // actualBuilder.append(resultSet.getString(i)).append(","); + // expectedBuilder + // .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + // .append(","); + // } + // Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + // cnt++; + // } + // Assert.assertEquals(retArray.length, cnt); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupBytimeTest() { + // String[] retArray = + // new String[] { + // "2,root.vehicle.d0,1,1,3,0,0,", + // "22,root.vehicle.d0,0,0,0,0,0,", + // "42,root.vehicle.d0,0,0,0,0,0,", + // "2,root.vehicle.d1,0,null,null,null,null,", + // "22,root.vehicle.d1,0,null,null,null,null,", + // "42,root.vehicle.d1,0,null,null,null,null," + // }; + // + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // + // try (ResultSet resultSet = + // statement.executeQuery( + // "select count(*) from root.vehicle.** GROUP BY ([2,50),20ms) align by device")) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // List actualIndexToExpectedIndexList = + // checkHeader( + // resultSetMetaData, + // "time,Device,count(s0),count(s1),count(s2),count(s3),count(s4)", + // new int[] { + // Types.tIMESTAMP, + // Types.VARCHAR, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // Types.BIGINT, + // }); + // + // int cnt = 0; + // while (resultSet.next()) { + // String[] expectedStrings = retArray[cnt].split(","); + // StringBuilder expectedBuilder = new StringBuilder(); + // StringBuilder actualBuilder = new StringBuilder(); + // for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + // actualBuilder.append(resultSet.getString(i)).append(","); + // expectedBuilder + // .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + // .append(","); + // } + // Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + // cnt++; + // } + // Assert.assertEquals(retArray.length, cnt); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupBytimeWithValueFilterTest() { + // String[] retArray = + // new String[] { + // "2,root.vehicle.d0,2,", "102,root.vehicle.d0,1", + // }; + // + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // + // try (ResultSet resultSet = + // statement.executeQuery( + // "select count(s2) from root.vehicle.d0 where s2 > 3 and s2 <= 10 GROUP BY + // ([2,200),100ms) align by device")) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // List actualIndexToExpectedIndexList = + // checkHeader( + // resultSetMetaData, + // "time,Device,count(s2)", + // new int[] { + // Types.tIMESTAMP, Types.VARCHAR, Types.BIGINT, + // }); + // + // int cnt = 0; + // while (resultSet.next()) { + // String[] expectedStrings = retArray[cnt].split(","); + // StringBuilder expectedBuilder = new StringBuilder(); + // StringBuilder actualBuilder = new StringBuilder(); + // for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + // actualBuilder.append(resultSet.getString(i)).append(","); + // expectedBuilder + // .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + // .append(","); + // } + // Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + // cnt++; + // } + // Assert.assertEquals(retArray.length, cnt); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + + @Test + public void unusualCaseTest2() { + + String[] expectedHeader = + new String[] {"s0", "s0", "s1", "time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "101,101,1101,1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + "10000,10000,40000,1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "null,null,null,1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "null,null,null,1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "999,999,null,1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + }; + tableResultSetEqualTest( + "select s0,s0,s1,* from vehicle where time < 20 and (device_id='d0' or device_id='d1') order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWithRegularExpressionTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + "1970-01-01T00:00:00.002Z,d0,10000,40000,2.22,null,null,", + "1970-01-01T00:00:00.003Z,d0,null,null,3.33,null,null,", + "1970-01-01T00:00:00.004Z,d0,null,null,4.44,null,null,", + "1970-01-01T00:00:00.050Z,d0,10000,50000,null,null,null,", + "1970-01-01T00:00:00.060Z,d0,null,null,null,aaaaa,null,", + "1970-01-01T00:00:00.070Z,d0,null,null,null,bbbbb,null,", + "1970-01-01T00:00:00.080Z,d0,null,null,null,ccccc,null,", + "1970-01-01T00:00:00.100Z,d0,99,199,null,null,true,", + "1970-01-01T00:00:00.101Z,d0,99,199,null,ddddd,null,", + "1970-01-01T00:00:00.102Z,d0,80,180,10.0,fffff,null,", + "1970-01-01T00:00:00.103Z,d0,99,199,null,null,null,", + "1970-01-01T00:00:00.104Z,d0,90,190,null,null,null,", + "1970-01-01T00:00:00.105Z,d0,99,199,11.11,null,null,", + "1970-01-01T00:00:00.106Z,d0,99,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d0,22222,55555,1000.11,null,null,", + "1999-12-31T16:00:00.000Z,d0,null,100,null,good,null,", + "1970-01-01T00:00:00.001Z,d1,999,null,null,null,null,", + "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle where device_id like 'd%' order by device_id", + expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void selectWithNonExistMeasurementInWhereClause() { + + String[] expectedHeader = new String[] {"time", "device_id", "s0", "s1", "s2", "s3", "s4"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d0,101,1101,null,null,null,", + }; + tableResultSetEqualTest( + "select * from vehicle where s1=1101 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableView2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableView2IT.java new file mode 100644 index 0000000000000..b422a552b803a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableView2IT.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceWithTemplateTableView2IT + extends IoTDBAlignByDeviceWithTemplateTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableViewIT.java new file mode 100644 index 0000000000000..668cd36a71a01 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableViewIT.java @@ -0,0 +1,543 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignByDeviceWithTemplateTableViewIT { + + private static final String DATABASE_NAME = "db"; + private static final String[] sqls = + new String[] { + // non-aligned template + "CREATE database root.sg1;", + "CREATE schema template t1 (s1 FLOAT encoding=RLE, s2 BOOLEAN encoding=PLAIN compression=SNAPPY, s3 INT32);", + "SET SCHEMA TEMPLATE t1 to root.sg1;", + "INSERT INTO root.sg1.d1(timestamp,s1,s2,s3) values(1,1.1,false,1), (2,2.2,false,2);", + "INSERT INTO root.sg1.d2(timestamp,s1,s2,s3) values(1,11.1,false,11), (2,22.2,false,22);", + "INSERT INTO root.sg1.d3(timestamp,s1,s2,s3) values(1,111.1,true,null), (4,444.4,true,44);", + "INSERT INTO root.sg1.d4(timestamp,s1,s2,s3) values(1,1111.1,true,1111), (5,5555.5,false,5555);", + + // aligned template + "CREATE database root.sg2;", + "CREATE schema template t2 aligned (s1 FLOAT encoding=RLE, s2 BOOLEAN encoding=PLAIN compression=SNAPPY, s3 INT32);", + "SET SCHEMA TEMPLATE t2 to root.sg2;", + "INSERT INTO root.sg2.d1(timestamp,s1,s2,s3) values(1,1.1,false,1), (2,2.2,false,2);", + "INSERT INTO root.sg2.d2(timestamp,s1,s2,s3) values(1,11.1,false,11), (2,22.2,false,22);", + "INSERT INTO root.sg2.d3(timestamp,s1,s2,s3) values(1,111.1,true,null), (4,444.4,true,44);", + "INSERT INTO root.sg2.d4(timestamp,s1,s2,s3) values(1,1111.1,true,1111), (5,5555.5,false,5555);", + }; + + private static final String[] createTableViewSqls = + new String[] { + "CREATE database " + DATABASE_NAME, + "use " + DATABASE_NAME, + "create view table1(device_id STRING TAG, s1 FLOAT FIELD, s2 BOOLEAN FIELD, s3 INT32 FIELD) as root.sg2.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void selectWildcardNoFilterTest() { + // 1. order by device_id + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,1.1,false,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,false,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,false,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,false,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,true,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,true,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,true,", + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s1, s2 FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3", "s1"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,1.1,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,2.2,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,11.1,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,22.2,", + "1970-01-01T00:00:00.001Z,d3,111.1,true,null,111.1,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,444.4,", + "1970-01-01T00:00:00.001Z,d4,1111.1,true,1111,1111.1,", + "1970-01-01T00:00:00.005Z,d4,5555.5,false,5555,5555.5,", + }; + tableResultSetEqualTest( + "SELECT *, s1 FROM table1 order by device_id", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 where device_id = 'd1' or device_id = 'd2' or device_id = 'd6' order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by device_id + limit/offset + expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 order by device_id, time OFFSET 1 LIMIT 2", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by time + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555.5,false,5555,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + "1970-01-01T00:00:00.001Z,d3,111.1,true,null,", + "1970-01-01T00:00:00.001Z,d4,1111.1,true,1111,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY time DESC, device_id ASC", + expectedHeader, + retArray, + DATABASE_NAME); + + // 4. order by time + limit/offset + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555.5,false,5555,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY time DESC, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectMeasurementNoFilterTest() { + // 1. order by device_id + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,1.1,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,", + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s1 FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery("SELECT s3,s1,s_null FROM table1 order by device_id")) { + fail("should throw exception to indicate that s_null doesn't exist"); + } catch (SQLException e) { + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // 2. order by device + limit/offset + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,2.2,", "1970-01-01T00:00:00.001Z,d2,11,11.1,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s1 FROM table1 order by device_id,time OFFSET 1 LIMIT 2", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by time + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,", + "1970-01-01T00:00:00.001Z,d1,1,1.1,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s1 FROM table1 ORDER BY TIME DESC, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 4. order by time + limit/offset + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s1 FROM table1 ORDER BY time DESC, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectWildcardWithFilterOrderByTimeTest() { + // 1. order by time + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d1,2.2,false,2,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + "1970-01-01T00:00:00.001Z,d1,1.1,false,1,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE time < 5 ORDER BY TIME desc, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by time + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY time DESC, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by time + value filter: s_null > 1 + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery("SELECT * FROM table1 WHERE s_null > 1 order by device_id")) { + fail("should throw exception to indicate that s_null doesn't exist"); + } catch (SQLException e) { + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectWildcardWithFilterOrderByDeviceTest() { + // 1. order by device + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d4,1111.1,true,1111,", + "1970-01-01T00:00:00.001Z,d3,111.1,true,null,", + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.001Z,d2,11.1,false,11,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE time < 5 ORDER BY device_id DESC, time LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by device + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,444.4,true,44,", + "1970-01-01T00:00:00.002Z,d2,22.2,false,22,", + }; + tableResultSetEqualTest( + "SELECT * FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY device_id DESC", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectMeasurementWithFilterOrderByTimeTest() { + // 1. order by time + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,44,true,", + "1970-01-01T00:00:00.002Z,d1,2,false,", + "1970-01-01T00:00:00.002Z,d2,22,false,", + "1970-01-01T00:00:00.001Z,d1,1,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s2 FROM table1 WHERE time < 5 ORDER BY time DESC, device_id LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by time + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,44,true,", "1970-01-01T00:00:00.002Z,d2,22,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s2 FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY time DESC, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void selectMeasurementWithFilterOrderByDeviceTest() { + // 1. order by device + time filter + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d4,1111,true,", + "1970-01-01T00:00:00.001Z,d3,null,true,", + "1970-01-01T00:00:00.004Z,d3,44,true,", + "1970-01-01T00:00:00.001Z,d2,11,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s2 FROM table1 WHERE time < 5 ORDER BY device_id DESC, time LIMIT 4", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by device + time filter + value filter + retArray = + new String[] { + "1970-01-01T00:00:00.004Z,d3,44,true,", "1970-01-01T00:00:00.002Z,d2,22,false,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3,s2 FROM table1 where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " + + "ORDER BY device_id DESC, time asc", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void aliasTest() { + String[] expectedHeader = new String[] {"aa", "bb", "s3", "s2", "time", "device_id"}; + + String[] retArray = + new String[] { + "1.1,false,1,false,1970-01-01T00:00:00.001Z,d1,", + "2.2,false,2,false,1970-01-01T00:00:00.002Z,d1,", + "11.1,false,11,false,1970-01-01T00:00:00.001Z,d2,", + "22.2,false,22,false,1970-01-01T00:00:00.002Z,d2,", + "111.1,true,null,true,1970-01-01T00:00:00.001Z,d3,", + "444.4,true,44,true,1970-01-01T00:00:00.004Z,d3,", + "1111.1,true,1111,true,1970-01-01T00:00:00.001Z,d4,", + "5555.5,false,5555,false,1970-01-01T00:00:00.005Z,d4,", + }; + tableResultSetEqualTest( + "SELECT s1 as aa, s2 as bb, s3, s2, time, device_id FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "a", "b"}; + + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1.1,1.1,", + "1970-01-01T00:00:00.002Z,d1,2.2,2.2,", + "1970-01-01T00:00:00.001Z,d2,11.1,11.1,", + "1970-01-01T00:00:00.002Z,d2,22.2,22.2,", + "1970-01-01T00:00:00.001Z,d3,111.1,111.1,", + "1970-01-01T00:00:00.004Z,d3,444.4,444.4,", + "1970-01-01T00:00:00.001Z,d4,1111.1,1111.1,", + "1970-01-01T00:00:00.005Z,d4,5555.5,5555.5,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s1 as a, s1 as b FROM table1 order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void orderByExpressionTest() { + // 1. order by basic measurement + String[] expectedHeader = new String[] {"time", "device_id", "s3", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,5555.5,false,", + "1970-01-01T00:00:00.002Z,d2,22,22.2,false,", + "1970-01-01T00:00:00.001Z,d2,11,11.1,false,", + "1970-01-01T00:00:00.002Z,d1,2,2.2,false,", + "1970-01-01T00:00:00.001Z,d1,1,1.1,false,", + "1970-01-01T00:00:00.001Z,d4,1111,1111.1,true,", + "1970-01-01T00:00:00.004Z,d3,44,444.4,true,", + "1970-01-01T00:00:00.001Z,d3,null,111.1,true,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3, s1, s2 FROM table1 order by s2 asc, s1 desc, device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. select measurement is different with order by measurement + expectedHeader = new String[] {"time", "device_id", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,", + "1970-01-01T00:00:00.002Z,d2,22,", + "1970-01-01T00:00:00.001Z,d2,11,", + "1970-01-01T00:00:00.002Z,d1,2,", + "1970-01-01T00:00:00.001Z,d1,1,", + "1970-01-01T00:00:00.001Z,d4,1111,", + "1970-01-01T00:00:00.004Z,d3,44,", + "1970-01-01T00:00:00.001Z,d3,null,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3 FROM table1 order by s2 asc, s1 desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. order by expression + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d4,5555,", + "1970-01-01T00:00:00.001Z,d4,1111,", + "1970-01-01T00:00:00.004Z,d3,44,", + "1970-01-01T00:00:00.002Z,d2,22,", + "1970-01-01T00:00:00.001Z,d2,11,", + "1970-01-01T00:00:00.002Z,d1,2,", + "1970-01-01T00:00:00.001Z,d1,1,", + "1970-01-01T00:00:00.001Z,d3,null,", + }; + tableResultSetEqualTest( + "SELECT time, device_id, s3 FROM table1 order by s1+s3 desc", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void templateInvalidTest() { + + tableAssertTestFail( + "select s1 from table1 where s1", + "701: WHERE clause must evaluate to a boolean: actual type FLOAT", + DATABASE_NAME); + } + + @Test + public void emptyResultTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3"}; + + String[] retArray = new String[] {}; + tableResultSetEqualTest( + "SELECT * FROM table1 where time>=now()-1d and time<=now() " + "ORDER BY time DESC", + expectedHeader, + retArray, + DATABASE_NAME); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : createTableViewSqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableView2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableView2IT.java new file mode 100644 index 0000000000000..dcecb8918ecfd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableView2IT.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByLimitOffsetAlignByDeviceTableView2IT + extends IoTDBOrderByLimitOffsetAlignByDeviceTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData3(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableViewIT.java new file mode 100644 index 0000000000000..6d913a26d2453 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceTableViewIT.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByLimitOffsetAlignByDeviceTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData3(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void orderByCanNotPushLimitTest() { + // 1. value filter, can not push down LIMIT + String[] expectedHeader = new String[] {"time", "device_id", "s1"}; + String[] retArray = new String[] {"1970-01-01T00:00:00.003Z,d1,111,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE s1>40 ORDER BY Time, device_id LIMIT 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // 2. order by expression, can not push down LIMIT + retArray = new String[] {"1970-01-01T00:00:00.003Z,d3,333,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY s1 DESC LIMIT 1", expectedHeader, retArray, DATABASE_NAME); + + // 3. time filter, can push down LIMIT + retArray = new String[] {"1970-01-01T00:00:00.002Z,d3,33,", "1970-01-01T00:00:00.002Z,d2,22,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE time>1 and time<3 ORDER BY device_id DESC,time LIMIT 2", + expectedHeader, + retArray, + DATABASE_NAME); + + // 4. both exist OFFSET and LIMIT, can push down LIMIT as OFFSET+LIMIT + retArray = new String[] {"1970-01-01T00:00:00.003Z,d2,222,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 ORDER BY Time DESC, device_id OFFSET 1 LIMIT 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // + // @Test + // public void aggregationWithHavingTest() { + // // when aggregation with having, can only use MergeSortNode but not use TopKNode + // String[] expectedHeader = new String[] {"Time,Device,sum(s1)"}; + // String[] retArray = new String[] {"3,root.db.d2,222.0,", "3,root.db.d3,333.0,"}; + // resultSetEqualTest( + // "select sum(s1) from root.db.** group by ((1,5],1ms) having(sum(s1)>111) order by time + // limit 2 align by device", + // expectedHeader, + // retArray); + // } + // + // @Test + // public void fillTest() { + // // linear fill can not use TopKNode + // String[] expectedHeader = new String[] {"Time,Device,s1,s2"}; + // String[] retArray = + // new String[] { + // "1,root.fill.d1,1,null,", + // "1,root.fill.d2,2,null,", + // "1,root.fill.d3,3,null,", + // "2,root.fill.d1,22,11.0,", + // }; + // resultSetEqualTest( + // "select * from root.fill.** order by time fill(linear) limit 4 align by device", + // expectedHeader, + // retArray); + // } + + private static final String DATABASE_NAME = "db"; + + private static final String DATABASE_FILL_NAME = "fill1"; + + private static final String[] SQL_LIST = + new String[] { + "CREATE DATABASE root.db;", + "CREATE TIMESERIES root.db.d1.s1 WITH DATATYPE=INT32, ENCODING=RLE;", + "CREATE TIMESERIES root.db.d2.s1 WITH DATATYPE=INT32, ENCODING=RLE;", + "CREATE TIMESERIES root.db.d3.s1 WITH DATATYPE=INT32, ENCODING=RLE;", + "INSERT INTO root.db.d1(timestamp,s1) VALUES(1, 1), (2, 11), (3, 111);", + "INSERT INTO root.db.d2(timestamp,s1) VALUES(1, 2), (2, 22), (3, 222);", + "INSERT INTO root.db.d3(timestamp,s1) VALUES(1, 3), (2, 33), (3, 333);", + "CREATE DATABASE root.fill1;", + "CREATE TIMESERIES root.fill1.d1.s1 WITH DATATYPE=INT32, ENCODING=RLE;", + "CREATE TIMESERIES root.fill1.d1.s2 WITH DATATYPE=FLOAT, ENCODING=RLE;", + "CREATE TIMESERIES root.fill1.d2.s1 WITH DATATYPE=INT32, ENCODING=RLE;", + "CREATE TIMESERIES root.fill1.d2.s2 WITH DATATYPE=FLOAT, ENCODING=RLE;", + "CREATE TIMESERIES root.fill1.d3.s1 WITH DATATYPE=INT32, ENCODING=RLE;", + "CREATE TIMESERIES root.fill1.d3.s2 WITH DATATYPE=FLOAT, ENCODING=RLE;", + "INSERT INTO root.fill1.d1(timestamp,s1,s2) VALUES(1, 1, null), (2, null, 11), (3, 111, 111.1);", + "INSERT INTO root.fill1.d2(timestamp,s1,s2) VALUES(1, 2, null), (2, 22, 22.2), (3, 222, null);", + "INSERT INTO root.fill1.d3(timestamp,s1,s2) VALUES(1, 3, null), (2, 33, null), (3, 333, 333.3);", + }; + + private static final String[] CREATE_TABLE_VIEW_SQL_LIST = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create view table1(device_id STRING TAG, s1 INT32 FIELD) as root.db.**", + "CREATE DATABASE " + DATABASE_FILL_NAME, + "USE " + DATABASE_FILL_NAME, + "create view table1(device_id STRING TAG, s1 INT32 FIELD, s2 FLOAT FIELD) as root.fill1.**", + }; + + protected static void insertData3() { + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + for (String sql : SQL_LIST) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + for (String sql : CREATE_TABLE_VIEW_SQL_LIST) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableView2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableView2IT.java new file mode 100644 index 0000000000000..399a7ab7b7433 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableView2IT.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByWithAlignByDeviceTableView2IT + extends IoTDBOrderByWithAlignByDeviceTableViewIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + insertData2(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableView3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableView3IT.java new file mode 100644 index 0000000000000..f3c016f7d6a17 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableView3IT.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByWithAlignByDeviceTableView3IT + extends IoTDBOrderByWithAlignByDeviceTableViewIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSeriesSlotNum(1); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + insertData2(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java new file mode 100644 index 0000000000000..d8bca2b4522d8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java @@ -0,0 +1,1588 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.alignbydevice; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.RpcUtils; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.db.it.utils.TestUtils.DEFAULT_ZONE_ID; +import static org.apache.iotdb.db.it.utils.TestUtils.TIME_PRECISION_IN_MS; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.rpc.RpcUtils.DEFAULT_TIME_FORMAT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByWithAlignByDeviceTableViewIT { + protected static final String DATABASE_NAME = "db"; + protected static final String[] places = + new String[] { + "London", + "Edinburgh", + "Belfast", + "Birmingham", + "Liverpool", + "Derby", + "Durham", + "Hereford", + "Manchester", + "Oxford" + }; + protected static final long startPrecipitation = 200; + protected static final double startTemperature = 20.0; + protected static final long startTime = 1668960000000L; + protected static final int numOfPointsInDevice = 20; + protected static final long timeGap = 100L; + protected static final Map deviceToStartTimestamp = new HashMap<>(); + public static final Map deviceToMaxTemperature = new HashMap<>(); + public static final Map deviceToAvgTemperature = new HashMap<>(); + public static final Map deviceToMaxPrecipitation = new HashMap<>(); + public static final Map deviceToAvgPrecipitation = new HashMap<>(); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + insertData2(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /** + * This method generate test data with crossing time. + * + *

The data can be viewed in online doc: + * + *

https://docs.google.com/spreadsheets/d/18XlOIi27ZIIdRnar2WNXVMxkZwjgwlPZmzJLVpZRpAA/edit#gid=0 + */ + protected static void insertData() { + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + // create TimeSeries + String prefix = "root.weather."; + for (String place : places) { + String PRE_PRECIPITATION = prefix + place + ".precipitation"; + String PRE_TEMPERATURE = prefix + place + ".temperature"; + String createPrecipitationSql = + "CREATE TIMESERIES " + PRE_PRECIPITATION + " WITH DATATYPE=INT64, ENCODING=RLE"; + String createTemperatureSql = + "CREATE TIMESERIES " + PRE_TEMPERATURE + " WITH DATATYPE=DOUBLE, ENCODING=RLE"; + statement.execute(createPrecipitationSql); + statement.execute(createTemperatureSql); + } + // insert data + long start = startTime; + double[][] temperatures = new double[places.length][29]; + long[][] precipitations = new long[places.length][29]; + for (int index = 0; index < places.length; index++) { + String place = places[index]; + + for (int i = 0; i < numOfPointsInDevice; i++) { + long precipitation = startPrecipitation + place.hashCode() + (start + i * timeGap); + double temperature = startTemperature + place.hashCode() + (start + i * timeGap); + precipitations[index][(int) ((start - startTime) / timeGap) + i] = precipitation; + temperatures[index][(int) ((start - startTime) / timeGap) + i] = temperature; + String insertUniqueTime = + "INSERT INTO " + + prefix + + place + + "(timestamp,precipitation,temperature) VALUES(" + + (start + i * timeGap) + + "," + + precipitation + + "," + + temperature + + ")"; + statement.execute(insertUniqueTime); + if (i == 0) deviceToStartTimestamp.put(place, start); + } + start += timeGap; + } + + for (int i = 0; i < places.length; i++) { + double[] aT = new double[3]; + double[] aP = new double[3]; + double[] mT = new double[3]; + long[] mP = new long[3]; + double totalTemperature = 0; + long totalPrecipitation = 0; + double maxTemperature = -1; + long maxPrecipitation = -1; + int cnt = 0; + for (int j = 0; j < precipitations[i].length; j++) { + totalTemperature += temperatures[i][j]; + totalPrecipitation += precipitations[i][j]; + maxPrecipitation = Math.max(maxPrecipitation, precipitations[i][j]); + maxTemperature = Math.max(maxTemperature, temperatures[i][j]); + if ((j + 1) % 10 == 0 || j == precipitations[i].length - 1) { + aT[cnt] = totalTemperature / 10; + aP[cnt] = (double) totalPrecipitation / 10; + mP[cnt] = maxPrecipitation; + mT[cnt] = maxTemperature; + cnt++; + totalTemperature = 0; + totalPrecipitation = 0; + maxTemperature = -1; + maxPrecipitation = -1; + } + } + deviceToMaxTemperature.put(places[i], mT); + deviceToMaxPrecipitation.put(places[i], mP); + deviceToAvgTemperature.put(places[i], aT); + deviceToAvgPrecipitation.put(places[i], aP); + } + + for (String sql : optimizedSQL) { + statement.execute(sql); + } + + } catch (Exception e) { + e.printStackTrace(); + } + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + statement.execute("create database " + DATABASE_NAME); + statement.execute("use " + DATABASE_NAME); + + statement.execute( + "create view weather(city STRING TAG, precipitation INT64 FIELD, temperature DOUBLE FIELD) as root.weather.**"); + + statement.execute( + "create view optimize(plant_id STRING TAG, device_id STRING TAG, temperature DOUBLE FIELD, status BOOLEAN FIELD, hardware STRING FIELD) as root.ln.**"); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + // use to test if the compare result of time will overflow. + protected static void insertData2() { + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + long startTime = 1; + String createSql = "CREATE TIMESERIES root.overflow.value WITH DATATYPE=INT32, ENCODING=RLE"; + statement.execute(createSql); + for (int i = 0; i < 20; i++) { + String insertTime = + "INSERT INTO " + + "root.overflow" + + "(timestamp,value) VALUES(" + + (startTime + 2147483648L) + + "," + + i + + ")"; + statement.execute(insertTime); + startTime += 2147483648L; + } + } catch (Exception e) { + e.printStackTrace(); + } + try (Connection iotDBConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = iotDBConnection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + statement.execute( + "create view overflow(device_id STRING TAG, value INT32 FIELD) as root.overflow.**"); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void overFlowTest() { + + String[] expectedHeader = new String[] {"time", "value"}; + String[] retArray = new String[20]; + + long startTime = 1; + for (int i = 0; i < 20; i++) { + long time = startTime + 2147483648L; + retArray[i] = + String.format( + "%s,%d,", + RpcUtils.formatDatetime( + DEFAULT_TIME_FORMAT, TIME_PRECISION_IN_MS, time, DEFAULT_ZONE_ID), + i); + startTime += 2147483648L; + } + + tableResultSetEqualTest( + "SELECT time, value FROM overflow", expectedHeader, retArray, DATABASE_NAME); + } + + // ORDER BY DEVICE + @Test + public void orderByDeviceTest1() { + + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery("SELECT * FROM weather order by city")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(10, index); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void orderByDeviceTest2() { + + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + try (ResultSet resultSet = + statement.executeQuery("SELECT * FROM weather order by city asc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(10, index); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void orderByDeviceTest3() { + + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + String[] expectedDevice = + Arrays.stream(places.clone()).sorted(Comparator.reverseOrder()).toArray(String[]::new); + + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + try (ResultSet resultSet = + statement.executeQuery("SELECT * FROM weather order by city desc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(10, index); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ORDER BY TIME + + @Test + public void orderByTimeTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeTest1( + "SELECT * FROM weather ORDER BY time, city", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeTest1("SELECT * FROM weather ORDER BY time, city LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeTest2() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + orderByTimeTest1( + "SELECT * FROM weather ORDER BY TIME ASC, city", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeTest1( + "SELECT * FROM weather ORDER BY time ASC, city ASC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeTest3() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + orderByTimeTest3( + "SELECT * FROM weather ORDER BY time DESC, city ASC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeTest3( + "SELECT * FROM weather ORDER BY time DESC, city LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeExpressionTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeExpressionTest1( + "SELECT * FROM weather ORDER BY time DESC, precipitation DESC, city", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeExpressionTest1( + "SELECT * FROM weather ORDER BY time DESC, precipitation DESC, city asc LIMIT 100", + 100, + expectedHeader); + } + + @Test + public void orderExpressionTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByExpressionTest1( + "SELECT * FROM weather ORDER BY precipitation DESC, time DESC, city asc", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByExpressionTest1( + "SELECT * FROM weather ORDER BY precipitation DESC, time DESC, city limit 100", + 100, + expectedHeader); + } + + @Test + public void orderByDeviceTimeTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest1("SELECT * FROM weather ORDER BY city ASC,time DESC", 10, expectedHeader); + + orderByDeviceTimeTest1( + "SELECT * FROM weather ORDER BY city ASC,time DESC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByDeviceTimeTest2() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest2("SELECT * FROM weather ORDER BY city ASC,time ASC", 10, expectedHeader); + + orderByDeviceTimeTest2( + "SELECT * FROM weather ORDER BY city ASC,Time ASC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByDeviceTimeTest3() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest3( + "SELECT * FROM weather ORDER BY city DESC,time DESC", 10, expectedHeader); + + orderByDeviceTimeTest3( + "SELECT * FROM weather ORDER BY city DESC,time DESC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByDeviceTimeTest4() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByDeviceTimeTest4("SELECT * FROM weather ORDER BY city DESC,time ASC", 10, expectedHeader); + + orderByDeviceTimeTest4( + "SELECT * FROM weather ORDER BY city DESC,time ASC LIMIT 100", 5, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest1() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest1( + "SELECT * FROM weather ORDER BY time ASC,city DESC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest1( + "SELECT * FROM weather ORDER BY time ASC,city DESC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest2() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest2( + "SELECT * FROM weather ORDER BY time ASC,city ASC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest2( + "SELECT * FROM weather ORDER BY time ASC,city ASC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest3() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest3( + "SELECT * FROM weather ORDER BY time DESC,city DESC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest3( + "SELECT * FROM weather ORDER BY time DESC,city DESC LIMIT 100", 100, expectedHeader); + } + + @Test + public void orderByTimeDeviceTest4() { + String[] expectedHeader = new String[] {"time", "city", "precipitation", "temperature"}; + + orderByTimeDeviceTest4( + "SELECT * FROM weather ORDER BY time DESC,city ASC", + numOfPointsInDevice * places.length, + expectedHeader); + + orderByTimeDeviceTest4( + "SELECT * FROM weather ORDER BY time DESC,city ASC LIMIT 100", 100, expectedHeader); + } + + public static void orderByTimeTest1(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = -1; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp >= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.isEmpty() && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeTest3(String sql, int count, String[] expectedHeader) { + + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeExpressionTest1(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = Long.MAX_VALUE; + long lastPrecipitation = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + if (actualDevice.equals(lastDevice) && actualTimeStamp == lastTimeStamp) { + assertTrue(actualPrecipitation <= lastPrecipitation); + lastPrecipitation = actualPrecipitation; + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByExpressionTest1(String sql, int count, String[] expectedHeader) { + + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = Long.MAX_VALUE; + long lastPrecipitation = Long.MAX_VALUE; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + assertTrue(actualPrecipitation <= lastPrecipitation); + if (actualPrecipitation == lastPrecipitation) { + assertTrue(actualTimeStamp <= lastTimeStamp); + } + lastPrecipitation = actualPrecipitation; + lastTimeStamp = actualTimeStamp; + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ORDER BY DEVICE,TIME + public static void orderByDeviceTimeTest1(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals( + deviceToStartTimestamp.get(actualDevice) + timeGap * (numOfPointsInDevice - cnt - 1), + actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByDeviceTimeTest2(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = Arrays.stream(places.clone()).sorted().toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByDeviceTimeTest3(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = + Arrays.stream(places.clone()).sorted(Comparator.reverseOrder()).toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals( + deviceToStartTimestamp.get(actualDevice) + timeGap * (numOfPointsInDevice - cnt - 1), + actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByDeviceTimeTest4(String sql, int count, String[] expectedHeader) { + String[] expectedDevice = + Arrays.stream(places.clone()).sorted(Comparator.reverseOrder()).toArray(String[]::new); + int index = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + assertEquals(expectedDevice[index], actualDevice); + assertEquals(deviceToStartTimestamp.get(actualDevice) + cnt * timeGap, actualTimeStamp); + + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + (startTemperature + actualDevice.hashCode() + actualTimeStamp) - actualTemperature + < 0.00001); + + cnt++; + if (cnt % numOfPointsInDevice == 0) { + index++; + cnt = 0; + } + } + assertEquals(count, index); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ORDER BY TIME,DEVICE + public static void orderByTimeDeviceTest1(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + long lastTimeStamp = -1; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp >= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) <= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeDeviceTest2(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + long lastTimeStamp = -1; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp >= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeDeviceTest3(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + long lastTimeStamp = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) <= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void orderByTimeDeviceTest4(String sql, int count, String[] expectedHeader) { + int total = 0; + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + long lastTimeStamp = Long.MAX_VALUE; + String lastDevice = ""; + while (resultSet.next()) { + long actualTimeStamp = resultSet.getLong(1); + assertTrue(actualTimeStamp <= lastTimeStamp); + String actualDevice = resultSet.getString(2); + if (!lastDevice.equals("") && actualTimeStamp == lastTimeStamp) { + assertTrue(actualDevice.compareTo(lastDevice) >= 0); + } + lastDevice = actualDevice; + lastTimeStamp = actualTimeStamp; + long actualPrecipitation = resultSet.getLong(3); + double actualTemperature = resultSet.getDouble(4); + assertEquals( + startPrecipitation + actualDevice.hashCode() + actualTimeStamp, actualPrecipitation); + assertTrue( + startTemperature + actualDevice.hashCode() + actualTimeStamp - actualTemperature + < 0.00001); + total++; + } + assertEquals(count, total); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // + // // aggregation query + // private static final int[][] countIn1000MSFiledWith100MSTimeGap = + // new int[][] { + // {10, 10, 0}, + // {9, 10, 1}, + // {8, 10, 2}, + // {7, 10, 3}, + // {6, 10, 4}, + // {5, 10, 5}, + // {4, 10, 6}, + // {3, 10, 7}, + // {2, 10, 8}, + // {1, 10, 9} + // }; + // + // private static int getCountNum(String device, int cnt) { + // int index = 0; + // for (int i = 0; i < places.length; i++) { + // if (places[i].equals(device)) { + // index = i; + // break; + // } + // } + // + // return countIn1000MSFiledWith100MSTimeGap[index][cnt]; + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest1() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000msC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime >= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt++; + // if (cnt % 3 == 0) { + // cnt = 0; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest2() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY DEVICE DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime >= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt++; + // if (cnt % 3 == 0) { + // cnt = 0; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest3() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY DEVICE + // DESC,TIME DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime <= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt--; + // if (cnt < 0) { + // cnt = 2; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByDeviceTest4() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY DEVICE + // ASC,TIME DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // long[] expectedTime = new long[] {startTime, startTime + 1000, startTime + 1000 * 2}; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertEquals(expectedTime[cnt], actualTime); + // if (!Objects.equals(lastDevice, "") && Objects.equals(actualDevice, lastDevice)) { + // assertTrue(actualTime <= lastTime); + // } + // if (!Objects.equals(lastDevice, "")) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // cnt--; + // if (cnt < 0) { + // cnt = 2; + // } + // } + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest1() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime >= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt++; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest2() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = Long.MAX_VALUE; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime <= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) >= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt--; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest3() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME + // ASC,DEVICE DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = -1; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 0; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime >= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt++; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // @Test + // public void groupByTimeOrderByTimeTest4() { + // String sql = + // "SELECT AVG(*),COUNT(*),MAX_VALUE(*) FROM weather GROUP + // BY([2022-11-21T00:00:00.000+08:00,2022-11-21T00:00:02.801+08:00),1000ms) ORDER BY TIME + // DESC,DEVICE DESC"; + // try (Connection connection = EnvFactory.getEnv().getConnection(); + // Statement statement = connection.createStatement()) { + // long lastTime = Long.MAX_VALUE; + // String lastDevice = ""; + // int index = 0; + // try (ResultSet resultSet = statement.executeQuery(sql)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader( + // resultSetMetaData, + // + // "Time,Device,AVG(precipitation),AVG(temperature),COUNT(precipitation),COUNT(temperature),MAX_VALUE(precipitation),MAX_VALUE(temperature)"); + // int cnt = 2; + // while (resultSet.next()) { + // long actualTime = resultSet.getLong(1); + // String actualDevice = resultSet.getString(2); + // assertTrue(actualTime <= lastTime); + // if (!Objects.equals(lastDevice, "") && actualTime == lastTime) { + // assertTrue(actualDevice.compareTo(lastDevice) <= 0); + // } + // lastTime = actualTime; + // lastDevice = actualDevice; + // double avgPrecipitation = resultSet.getDouble(3); + // assertTrue(deviceToAvgPrecipitation.get(actualDevice)[cnt] - avgPrecipitation < + // 0.00001); + // + // double avgTemperature = resultSet.getDouble(4); + // assertTrue(deviceToAvgTemperature.get(actualDevice)[cnt] - avgTemperature < 0.00001); + // + // int countPrecipitation = resultSet.getInt(5); + // assertEquals(getCountNum(actualDevice, cnt), countPrecipitation); + // int countTemperature = resultSet.getInt(6); + // assertEquals(getCountNum(actualDevice, cnt), countTemperature); + // + // long maxPrecipitation = resultSet.getLong(7); + // assertEquals(deviceToMaxPrecipitation.get(actualDevice)[cnt], maxPrecipitation); + // + // double maxTemperature = resultSet.getDouble(8); + // assertTrue(deviceToMaxTemperature.get(actualDevice)[cnt] - maxTemperature < 0.00001); + // index++; + // if (index % 10 == 0) { + // cnt--; + // } + // } + // assertEquals(30, index); + // } + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + + // test the optimized plan + public static String[] optimizedSQL = + new String[] { + "insert into root.ln.wf01.wt01(timestamp,temperature,status) values(2017-11-01T00:00:00.000+08:00,25.96,true)", + "insert into root.ln.wf01.wt01(timestamp,temperature,status) values(2017-11-01T00:01:00.000+08:00,24.36,true)", + "insert into root.ln.wf02.wt02(timestamp,status,hardware) values(1970-01-01T08:00:00.001+08:00,true,'v1')", + "insert into root.ln.wf02.wt02(timestamp,status,hardware) values(1970-01-01T08:00:00.002+08:00,false,'v2')", + "insert into root.ln.wf02.wt02(timestamp,status,hardware) values(2017-11-01T00:00:00.000+08:00,false,'v2')", + "insert into root.ln.wf02.wt02(timestamp,status,hardware) values(2017-11-01T00:01:00.000+08:00,true,'v2')" + }; + + @Test + public void optimizedPlanTest() { + + String[] expectedHeader = + new String[] {"time", "plant_id", "device_id", "temperature", "status", "hardware"}; + String[] retArray = + new String[] { + "2017-10-31T16:01:00.000Z,wf02,wt02,null,true,v2,", + "2017-10-31T16:01:00.000Z,wf01,wt01,24.36,true,null,", + "2017-10-31T16:00:00.000Z,wf02,wt02,null,false,v2,", + "2017-10-31T16:00:00.000Z,wf01,wt01,25.96,true,null,", + "1970-01-01T00:00:00.002Z,wf02,wt02,null,false,v2,", + "1970-01-01T00:00:00.001Z,wf02,wt02,null,true,v1,", + }; + + tableResultSetEqualTest( + "SELECT * FROM optimize ORDER BY Time DESC,plant_id DESC,device_id desc", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedOffsetLimitPushDownTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedOffsetLimitPushDownTableViewIT.java new file mode 100644 index 0000000000000..72cc9ebb4bb01 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedOffsetLimitPushDownTableViewIT.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.relational.it.query.view.old.aligned.TableViewUtils.USE_DB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedOffsetLimitPushDownTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(1,1,1)"); + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(2,2,2)"); + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(3,3,3)"); + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(4,4,4)"); + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(5,5,null)"); + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(6,6,null)"); + statement.addBatch("insert into root.db.d1(time,s1,s2) aligned values(7,7,7)"); + + statement.addBatch("flush"); + + statement.executeBatch(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS db"); + statement.execute(USE_DB); + statement.execute( + "CREATE VIEW table0 (device string tag, s1 double field, s2 double field) as root.db.**"); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void selectWithLimitPushDownTest1() { + + String[] retArray = new String[] {"3,3.0", "4,4.0"}; + + String[] columnNames = {"s1"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1 from table0 where device='d1' and time >= 3 limit 2")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectWithLimitPushDownTest2() { + + String[] retArray = new String[] {"6,6.0,null", "7,7.0,7.0"}; + + String[] columnNames = {"s1", "s2"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2 from table0 where device='d1' and time >= 1 offset 5")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView2IT.java new file mode 100644 index 0000000000000..3cc32d2c689d7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView2IT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTableView2IT extends IoTDBAlignedSeriesQueryTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(2); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView3IT.java new file mode 100644 index 0000000000000..61279a2adc125 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView3IT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTableView3IT extends IoTDBAlignedSeriesQueryTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView4IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView4IT.java new file mode 100644 index 0000000000000..da0dcc9d1ecfd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView4IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTableView4IT extends IoTDBAlignedSeriesQueryTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView5IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView5IT.java new file mode 100644 index 0000000000000..8cdd3c4cfb954 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableView5IT.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTableView5IT extends IoTDBAlignedSeriesQueryTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(1) + .setMaxNumberOfPointsInPage(1) + .setDriverTaskExecutionTimeSliceInMs(20); + + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableViewIT.java new file mode 100644 index 0000000000000..34051a1d26260 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBAlignedSeriesQueryTableViewIT.java @@ -0,0 +1,3698 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.db.utils.constant.TestConstant.avg; +import static org.apache.iotdb.db.utils.constant.TestConstant.count; +import static org.apache.iotdb.db.utils.constant.TestConstant.firstValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.lastValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.maxTime; +import static org.apache.iotdb.db.utils.constant.TestConstant.maxValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.minTime; +import static org.apache.iotdb.db.utils.constant.TestConstant.minValue; +import static org.apache.iotdb.db.utils.constant.TestConstant.sum; +import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; +import static org.apache.iotdb.relational.it.query.view.old.aligned.TableViewUtils.USE_DB; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlignedSeriesQueryTableViewIT { + + private static final double DELTA = 1e-6; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ------------------------------Raw Query Without Value Filter---------------------------------- + @Test + public void selectAllAlignedWithoutValueFilterTest() { + + String[] retArray = + new String[] { + "1,1.0,1,1,true,aligned_test1", + "2,2.0,2,2,null,aligned_test2", + "3,30000.0,null,30000,true,aligned_unseq_test3", + "4,4.0,4,null,true,aligned_test4", + "5,5.0,5,null,true,aligned_test5", + "6,6.0,6,6,true,null", + "7,7.0,7,7,false,aligned_test7", + "8,8.0,8,8,null,aligned_test8", + "9,9.0,9,9,false,aligned_test9", + "10,null,10,10,true,aligned_test10", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + "21,null,null,21,true,null", + "22,null,null,22,true,null", + "23,230000.0,null,230000,false,null", + "24,null,null,24,true,null", + "25,null,null,25,true,null", + "26,null,null,26,false,null", + "27,null,null,27,false,null", + "28,null,null,28,false,null", + "29,null,null,29,false,null", + "30,null,null,30,false,null", + "31,null,31,null,null,aligned_test31", + "32,null,32,null,null,aligned_test32", + "33,null,33,null,null,aligned_test33", + "34,null,34,null,null,aligned_test34", + "35,null,35,null,null,aligned_test35", + "36,null,36,null,null,aligned_test36", + "37,null,37,null,null,aligned_test37", + "38,null,38,null,null,aligned_test38", + "39,null,39,null,null,aligned_test39", + "40,null,40,null,null,aligned_test40", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeFilterTest() { + + String[] retArray = + new String[] { + "9,9.0,9,9,false,aligned_test9", + "10,null,10,10,true,aligned_test10", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + "21,null,null,21,true,null", + "22,null,null,22,true,null", + "23,230000.0,null,230000,false,null", + "24,null,null,24,true,null", + "25,null,null,25,true,null", + "26,null,null,26,false,null", + "27,null,null,27,false,null", + "28,null,null,28,false,null", + "29,null,null,29,false,null", + "30,null,null,30,false,null", + "31,null,31,null,null,aligned_test31", + "32,null,32,null,null,aligned_test32", + "33,null,33,null,null,aligned_test33", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterTest1() { + + String[] retArray = + new String[] { + "1,1.0,true,aligned_test1", + "2,2.0,null,aligned_test2", + "3,30000.0,true,aligned_unseq_test3", + "4,4.0,true,aligned_test4", + "5,5.0,true,aligned_test5", + "6,6.0,true,null", + "7,7.0,false,aligned_test7", + "8,8.0,null,aligned_test8", + "9,9.0,false,aligned_test9", + "10,null,true,aligned_test10", + "11,11.0,null,null", + "12,12.0,null,null", + "13,130000.0,true,aligned_unseq_test13", + "14,14.0,null,null", + "15,15.0,null,null", + "16,16.0,null,null", + "17,17.0,null,null", + "18,18.0,null,null", + "19,19.0,null,null", + "20,20.0,null,null", + "21,null,true,null", + "22,null,true,null", + "23,230000.0,false,null", + "24,null,true,null", + "25,null,true,null", + "26,null,false,null", + "27,null,false,null", + "28,null,false,null", + "29,null,false,null", + "30,null,false,null", + "31,null,null,aligned_test31", + "32,null,null,aligned_test32", + "33,null,null,aligned_test33", + "34,null,null,aligned_test34", + "35,null,null,aligned_test35", + "36,null,null,aligned_test36", + "37,null,null,aligned_test37", + "38,null,null,aligned_test38", + "39,null,null,aligned_test39", + "40,null,null,aligned_test40", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery("select Time,s1,s4,s5 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterTest2() { + + String[] retArray = + new String[] { + "1,1.0,true", + "2,2.0,null", + "3,30000.0,true", + "4,4.0,true", + "5,5.0,true", + "6,6.0,true", + "7,7.0,false", + "8,8.0,null", + "9,9.0,false", + "10,null,true", + "11,11.0,null", + "12,12.0,null", + "13,130000.0,true", + "14,14.0,null", + "15,15.0,null", + "16,16.0,null", + "17,17.0,null", + "18,18.0,null", + "19,19.0,null", + "20,20.0,null", + "21,null,true", + "22,null,true", + "23,230000.0,false", + "24,null,true", + "25,null,true", + "26,null,false", + "27,null,false", + "28,null,false", + "29,null,false", + "30,null,false", + "31,null,null", + "32,null,null", + "33,null,null", + "34,null,null", + "35,null,null", + "36,null,null", + "37,null,null", + "38,null,null", + "39,null,null", + "40,null,null", + }; + + String[] columnNames = {"s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery("select Time,s1,s4 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithTimeFilterTest() { + + String[] retArray = + new String[] { + "16,16.0,null,null", + "17,17.0,null,null", + "18,18.0,null,null", + "19,19.0,null,null", + "20,20.0,null,null", + "21,null,true,null", + "22,null,true,null", + "23,230000.0,false,null", + "24,null,true,null", + "25,null,true,null", + "26,null,false,null", + "27,null,false,null", + "28,null,false,null", + "29,null,false,null", + "30,null,false,null", + "31,null,null,aligned_test31", + "32,null,null,aligned_test32", + "33,null,null,aligned_test33", + "34,null,null,aligned_test34", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4,s5 from table0 where device='d1' and time >= 16 and time <= 34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ------------------------------Raw Query With Value Filter------------------------------------- + @Test + public void selectAllAlignedWithValueFilterTest1() { + + String[] retArray = + new String[] { + "1,1.0,1,1,true,aligned_test1", + "3,30000.0,null,30000,true,aligned_unseq_test3", + "4,4.0,4,null,true,aligned_test4", + "5,5.0,5,null,true,aligned_test5", + "6,6.0,6,6,true,null", + "10,null,10,10,true,aligned_test10", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "21,null,null,21,true,null", + "22,null,null,22,true,null", + "24,null,null,24,true,null", + "25,null,null,25,true,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterTest2() { + + String[] retArray = + new String[] { + "12,12.0,12,12,null,null", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and s1 > 11.0 and s2 <= 33 order by time asc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterTest3() { + + String[] retArray = + new String[] { + "1,1.0,1,1,true,aligned_test1", + "2,2.0,2,2,null,aligned_test2", + "3,30000.0,null,30000,true,aligned_unseq_test3", + "4,4.0,4,null,true,aligned_test4", + "5,5.0,5,null,true,aligned_test5", + "6,6.0,6,6,true,null", + "7,7.0,7,7,false,aligned_test7", + "8,8.0,8,8,null,aligned_test8", + "9,9.0,9,9,false,aligned_test9", + "10,null,10,10,true,aligned_test10", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + "19,19.0,19,19,null,null", + "20,20.0,20,20,null,null", + "23,230000.0,null,230000,false,null", + "31,null,31,null,null,aligned_test31", + "32,null,32,null,null,aligned_test32", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and (s1 >= 13.0 or s2 < 33) order by time asc")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeAndValueFilterTest1() { + + String[] retArray = + new String[] { + "9,9.0,9,9,false,aligned_test9", + "11,11.0,11,11,null,null", + "12,12.0,12,12,null,null", + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 and s1 < 19.0")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithLimitOffsetTest() { + + String[] retArray = + new String[] { + "14,14.0,14,14,null,null", + "15,15.0,15,15,null,null", + "16,16.0,16,16,null,null", + "17,17.0,17,17,null,null", + "18,18.0,18,18,null,null", + }; + + String[] columnNames = {"s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 offset 5 limit 5")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterTest1() { + + String[] retArray = + new String[] { + "1,1.0,true,aligned_test1", + "2,2.0,null,aligned_test2", + "4,4.0,true,aligned_test4", + "5,5.0,true,aligned_test5", + "6,6.0,true,null", + "7,7.0,false,aligned_test7", + "8,8.0,null,aligned_test8", + "9,9.0,false,aligned_test9", + "11,11.0,null,null", + "12,12.0,null,null", + "14,14.0,null,null", + "15,15.0,null,null", + "16,16.0,null,null", + "34,null,null,aligned_test34", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4,s5 from table0 where device='d1' and s1 < 17.0 or s5 = 'aligned_test34'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterTest2() { + + String[] retArray = + new String[] { + "7,7.0,false", "9,9.0,false", + }; + + String[] columnNames = {"s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4 from table0 where device='d1' and s1 < 19.0 and s4 = false")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithTimeAndValueFilterTest() { + + String[] retArray = + new String[] { + "23,230000.0,false,null", + "26,null,false,null", + "27,null,false,null", + "28,null,false,null", + "29,null,false,null", + "30,null,false,null", + }; + + String[] columnNames = {"s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,s1,s4,s5 from table0 where device='d1' and time >= 16 and time <= 34 and s4=false")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // --------------------------Aggregation Query Without Value Filter--------------------------- + @Ignore + @Test + public void countAllAlignedWithoutTimeFilterTest() { + String[] retArray = new String[] {"20", "29", "28", "19", "20"}; + String[] columnNames = { + "count(d1.s1)", "count(d1.s2)", "count(d1.s3)", "count(d1.s4)", "count(d1.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = statement.executeQuery("select count(*) from d1")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedAndNonAlignedWithoutTimeFilterTest() { + String[] retArray = new String[] {"20", "29", "28", "19", "20", "19", "29", "28", "18", "19"}; + String[] columnNames = { + "count(d1.s1)", + "count(d1.s2)", + "count(d1.s3)", + "count(d1.s4)", + "count(d1.s5)", + "count(d2.s1)", + "count(d2.s2)", + "count(d2.s3)", + "count(d2.s4)", + "count(d2.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = statement.executeQuery("select count(*) from root.sg1.*")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithTimeFilterTest() { + String[] retArray = new String[] {"12", "15", "22", "13", "6"}; + String[] columnNames = { + "count(d1.s1)", "count(d1.s2)", "count(d1.s3)", "count(d1.s4)", "count(d1.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 where time >= 9 and time <= 33")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** aggregate multi columns of aligned timeseries in one SQL */ + @Ignore + @Test + public void aggregateSomeAlignedWithoutTimeFilterTest() { + double[] retArray = + new double[] { + 20, 29, 28, 390184, 130549, 390417, 19509.2, 4501.689655172413, 13943.464285714286 + }; + String[] columnNames = { + "count(d1.s1)", + "count(d1.s2)", + "count(d1.s3)", + "sum(d1.s1)", + "sum(d1.s2)", + "sum(d1.s3)", + "avg(d1.s1)", + "avg(d1.s2)", + "avg(d1.s3)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1),count(s2),count(s3),sum(s1),sum(s2),sum(s3),avg(s1),avg(s2),avg(s3) from d1")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = Double.parseDouble(resultSet.getString(index)); + } + assertArrayEquals(retArray, ans, DELTA); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** aggregate multi columns of aligned timeseries in one SQL */ + @Ignore + @Test + public void aggregateSomeAlignedWithTimeFilterTest() { + double[] retArray = + new double[] { + 6, 9, 15, 230090, 220, 230322, 38348.333333333336, 24.444444444444443, 15354.8 + }; + String[] columnNames = { + "count(d1.s1)", + "count(d1.s2)", + "count(d1.s3)", + "sum(d1.s1)", + "sum(d1.s2)", + "sum(d1.s3)", + "avg(d1.s1)", + "avg(d1.s2)", + "avg(d1.s3)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1),count (s2),count (s3),sum(s1),sum(s2),sum(s3),avg(s1),avg(s2),avg(s3) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = Double.parseDouble(resultSet.getString(index)); + } + assertArrayEquals(retArray, ans, DELTA); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countSingleAlignedWithTimeFilterTest() { + String[] retArray = new String[] {"9"}; + String[] columnNames = {"count(d1.s2)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(s2) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + // No need to add time column for aggregation query + for (String columnName : columnNames) { + int index = map.get(columnName); + if (builder.length() != 0) { + builder.append(","); + } + builder.append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void sumSingleAlignedWithTimeFilterTest() { + String[] retArray = new String[] {"230322.0"}; + String[] columnNames = {"sum(d1.s3)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select sum(s3) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + // No need to add time column for aggregation query + for (String columnName : columnNames) { + int index = map.get(columnName); + if (builder.length() != 0) { + builder.append(","); + } + builder.append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void avgSingleAlignedWithTimeFilterTest() { + double[][] retArray = {{24.444444444444443}}; + String[] columnNames = {"avg(d1.s2)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select avg(s2) from d1 where time>=16 and time<=34")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + StringBuilder builder = new StringBuilder(); + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = Double.parseDouble(resultSet.getString(index)); + } + assertArrayEquals(retArray[cnt], ans, DELTA); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // --------------------------------Aggregation with value filter------------------------------ + @Ignore + @Test + public void countAlignedWithValueFilterTest() { + String[] retArray = new String[] {"11"}; + String[] columnNames = {"count(d1.s4)"}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(s4) from d1 where s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationFuncAlignedWithValueFilterTest() throws ClassNotFoundException { + String[] retArray = new String[] {"8", "42.0", "5.25", "1.0", "9.0", "1", "9", "1.0", "9.0"}; + String[] columnNames = { + "count(d1.s1)", + "sum(d1.s1)", + "avg(d1.s1)", + "first_value(d1.s1)", + "last_value(d1.s1)", + "min_time(d1.s1)", + "max_time(d1.s1)", + "min_value(d1.s1)", + "max_value(d1.s1)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s1), avg(s1), " + + "first_value(s1), last_value(s1), " + + "min_time(s1), max_time(s1)," + + "max_value(s1), min_value(s1) from d1 where s1 < 10")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithValueFilterTest() throws ClassNotFoundException { + String[] retArray = new String[] {"6", "6", "9", "11", "6"}; + String[] columnNames = { + "count(d1.s1)", "count(d1.s2)", "count(d1.s3)", "count(d1.s4)", "count(d1.s5)" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 where s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationAllAlignedWithValueFilterTest() throws ClassNotFoundException { + String[] retArray = new String[] {"160016.0", "11", "1", "13"}; + String[] columnNames = { + "sum(d1.s1)", "count(d1.s4)", "min_value(d1.s3)", "max_time(d1.s2)", + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select sum(s1), count(s4), min_value(s3), max_time(s2) from d1 where s4 = true")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + // ---------------------------------Align by device query ------------------------------------ + @Test + public void selectAllAlignedWithoutValueFilterAlignByDeviceTest() { + String[] retArray = + new String[] { + "1,d1,1.0,1,1,true,aligned_test1", + "2,d1,2.0,2,2,null,aligned_test2", + "3,d1,30000.0,null,30000,true,aligned_unseq_test3", + "4,d1,4.0,4,null,true,aligned_test4", + "5,d1,5.0,5,null,true,aligned_test5", + "6,d1,6.0,6,6,true,null", + "7,d1,7.0,7,7,false,aligned_test7", + "8,d1,8.0,8,8,null,aligned_test8", + "9,d1,9.0,9,9,false,aligned_test9", + "10,d1,null,10,10,true,aligned_test10", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "23,d1,230000.0,null,230000,false,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + "26,d1,null,null,26,false,null", + "27,d1,null,null,27,false,null", + "28,d1,null,null,28,false,null", + "29,d1,null,null,29,false,null", + "30,d1,null,null,30,false,null", + "31,d1,null,31,null,null,aligned_test31", + "32,d1,null,32,null,null,aligned_test32", + "33,d1,null,33,null,null,aligned_test33", + "34,d1,null,34,null,null,aligned_test34", + "35,d1,null,35,null,null,aligned_test35", + "36,d1,null,36,null,null,aligned_test36", + "37,d1,null,37,null,null,aligned_test37", + "38,d1,null,38,null,null,aligned_test38", + "39,d1,null,39,null,null,aligned_test39", + "40,d1,null,40,null,null,aligned_test40", + }; + + String[] columnNames = {"device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,device,s1,s2,s3,s4,s5 from table0 where device='d1' order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedAndNonAlignedAlignByDeviceTest() { + String[] retArray = + new String[] { + "1,d1,1.0,1,1,true,aligned_test1", + "2,d1,2.0,2,2,null,aligned_test2", + "3,d1,30000.0,null,30000,true,aligned_unseq_test3", + "4,d1,4.0,4,null,true,aligned_test4", + "5,d1,5.0,5,null,true,aligned_test5", + "6,d1,6.0,6,6,true,null", + "7,d1,7.0,7,7,false,aligned_test7", + "8,d1,8.0,8,8,null,aligned_test8", + "9,d1,9.0,9,9,false,aligned_test9", + "10,d1,null,10,10,true,aligned_test10", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "23,d1,230000.0,null,230000,false,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + "26,d1,null,null,26,false,null", + "27,d1,null,null,27,false,null", + "28,d1,null,null,28,false,null", + "29,d1,null,null,29,false,null", + "30,d1,null,null,30,false,null", + "31,d1,null,31,null,null,aligned_test31", + "32,d1,null,32,null,null,aligned_test32", + "33,d1,null,33,null,null,aligned_test33", + "34,d1,null,34,null,null,aligned_test34", + "35,d1,null,35,null,null,aligned_test35", + "36,d1,null,36,null,null,aligned_test36", + "37,d1,null,37,null,null,aligned_test37", + "38,d1,null,38,null,null,aligned_test38", + "39,d1,null,39,null,null,aligned_test39", + "40,d1,null,40,null,null,aligned_test40", + "1,d2,1.0,1,1,true,non_aligned_test1", + "2,d2,2.0,2,2,null,non_aligned_test2", + "3,d2,3.0,null,3,false,non_aligned_test3", + "4,d2,4.0,4,null,true,non_aligned_test4", + "5,d2,5.0,5,null,true,non_aligned_test5", + "6,d2,6.0,6,6,true,null", + "7,d2,7.0,7,7,false,non_aligned_test7", + "8,d2,8.0,8,8,null,non_aligned_test8", + "9,d2,9.0,9,9,false,non_aligned_test9", + "10,d2,null,10,10,true,non_aligned_test10", + "11,d2,11.0,11,11,null,null", + "12,d2,12.0,12,12,null,null", + "13,d2,13.0,13,13,null,null", + "14,d2,14.0,14,14,null,null", + "15,d2,15.0,15,15,null,null", + "16,d2,16.0,16,16,null,null", + "17,d2,17.0,17,17,null,null", + "18,d2,18.0,18,18,null,null", + "19,d2,19.0,19,19,null,null", + "20,d2,20.0,20,20,null,null", + "21,d2,null,null,21,true,null", + "22,d2,null,null,22,true,null", + "23,d2,null,null,23,true,null", + "24,d2,null,null,24,true,null", + "25,d2,null,null,25,true,null", + "26,d2,null,null,26,false,null", + "27,d2,null,null,27,false,null", + "28,d2,null,null,28,false,null", + "29,d2,null,null,29,false,null", + "30,d2,null,null,30,false,null", + "31,d2,null,31,null,null,non_aligned_test31", + "32,d2,null,32,null,null,non_aligned_test32", + "33,d2,null,33,null,null,non_aligned_test33", + "34,d2,null,34,null,null,non_aligned_test34", + "35,d2,null,35,null,null,non_aligned_test35", + "36,d2,null,36,null,null,non_aligned_test36", + "37,d2,null,37,null,null,non_aligned_test37", + "38,d2,null,38,null,null,non_aligned_test38", + "39,d2,null,39,null,null,non_aligned_test39", + "40,d2,null,40,null,null,non_aligned_test40", + }; + + String[] columnNames = {"device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,device,s1,s2,s3,s4,s5 from table0 order by device, time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeFilterAlignByDeviceTest() { + String[] retArray = + new String[] { + "9,d1,9.0,9,9,false,aligned_test9", + "10,d1,null,10,10,true,aligned_test10", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "23,d1,230000.0,null,230000,false,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + "26,d1,null,null,26,false,null", + "27,d1,null,null,27,false,null", + "28,d1,null,null,28,false,null", + "29,d1,null,null,29,false,null", + "30,d1,null,null,30,false,null", + "31,d1,null,31,null,null,aligned_test31", + "32,d1,null,32,null,null,aligned_test32", + "33,d1,null,33,null,null,aligned_test33", + }; + + String[] columnNames = {"device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,device,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "1,d1,1.0,true,aligned_test1", + "2,d1,2.0,null,aligned_test2", + "3,d1,30000.0,true,aligned_unseq_test3", + "4,d1,4.0,true,aligned_test4", + "5,d1,5.0,true,aligned_test5", + "6,d1,6.0,true,null", + "7,d1,7.0,false,aligned_test7", + "8,d1,8.0,null,aligned_test8", + "9,d1,9.0,false,aligned_test9", + "10,d1,null,true,aligned_test10", + "11,d1,11.0,null,null", + "12,d1,12.0,null,null", + "13,d1,130000.0,true,aligned_unseq_test13", + "14,d1,14.0,null,null", + "15,d1,15.0,null,null", + "16,d1,16.0,null,null", + "17,d1,17.0,null,null", + "18,d1,18.0,null,null", + "19,d1,19.0,null,null", + "20,d1,20.0,null,null", + "21,d1,null,true,null", + "22,d1,null,true,null", + "23,d1,230000.0,false,null", + "24,d1,null,true,null", + "25,d1,null,true,null", + "26,d1,null,false,null", + "27,d1,null,false,null", + "28,d1,null,false,null", + "29,d1,null,false,null", + "30,d1,null,false,null", + "31,d1,null,null,aligned_test31", + "32,d1,null,null,aligned_test32", + "33,d1,null,null,aligned_test33", + "34,d1,null,null,aligned_test34", + "35,d1,null,null,aligned_test35", + "36,d1,null,null,aligned_test36", + "37,d1,null,null,aligned_test37", + "38,d1,null,null,aligned_test38", + "39,d1,null,null,aligned_test39", + "40,d1,null,null,aligned_test40", + }; + + String[] columnNames = {"device", "s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery("select Time,device,s1,s4,s5 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithoutValueFilterAlignByDeviceTest2() { + String[] retArray = + new String[] { + "1,d1,1.0,true", + "2,d1,2.0,null", + "3,d1,30000.0,true", + "4,d1,4.0,true", + "5,d1,5.0,true", + "6,d1,6.0,true", + "7,d1,7.0,false", + "8,d1,8.0,null", + "9,d1,9.0,false", + "10,d1,null,true", + "11,d1,11.0,null", + "12,d1,12.0,null", + "13,d1,130000.0,true", + "14,d1,14.0,null", + "15,d1,15.0,null", + "16,d1,16.0,null", + "17,d1,17.0,null", + "18,d1,18.0,null", + "19,d1,19.0,null", + "20,d1,20.0,null", + "21,d1,null,true", + "22,d1,null,true", + "23,d1,230000.0,false", + "24,d1,null,true", + "25,d1,null,true", + "26,d1,null,false", + "27,d1,null,false", + "28,d1,null,false", + "29,d1,null,false", + "30,d1,null,false", + "31,d1,null,null", + "32,d1,null,null", + "33,d1,null,null", + "34,d1,null,null", + "35,d1,null,null", + "36,d1,null,null", + "37,d1,null,null", + "38,d1,null,null", + "39,d1,null,null", + "40,d1,null,null", + }; + + String[] columnNames = {"Device", "s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery("select Time,Device,s1,s4 from table0 where device='d1'")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithTimeFilterAlignByDeviceTest() { + String[] retArray = + new String[] { + "16,d1,16.0,null,null", + "17,d1,17.0,null,null", + "18,d1,18.0,null,null", + "19,d1,19.0,null,null", + "20,d1,20.0,null,null", + "21,d1,null,true,null", + "22,d1,null,true,null", + "23,d1,230000.0,false,null", + "24,d1,null,true,null", + "25,d1,null,true,null", + "26,d1,null,false,null", + "27,d1,null,false,null", + "28,d1,null,false,null", + "29,d1,null,false,null", + "30,d1,null,false,null", + "31,d1,null,null,aligned_test31", + "32,d1,null,null,aligned_test32", + "33,d1,null,null,aligned_test33", + "34,d1,null,null,aligned_test34", + }; + + String[] columnNames = {"Device", "s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s4,s5 from table0 where device='d1' and time >= 16 and time <= 34 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "1,d1,1.0,1,1,true,aligned_test1", + "3,d1,30000.0,null,30000,true,aligned_unseq_test3", + "4,d1,4.0,4,null,true,aligned_test4", + "5,d1,5.0,5,null,true,aligned_test5", + "6,d1,6.0,6,6,true,null", + "10,d1,null,10,10,true,aligned_test10", + "13,d1,130000.0,130000,130000,true,aligned_unseq_test13", + "21,d1,null,null,21,true,null", + "22,d1,null,null,22,true,null", + "24,d1,null,null,24,true,null", + "25,d1,null,null,25,true,null", + }; + + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s2,s3,s4,s5 from table0 where device='d1' and s4 = true order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithValueFilterAlignByDeviceTest2() { + String[] retArray = + new String[] { + "12,d1,12.0,12,12,null,null", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + "19,d1,19.0,19,19,null,null", + "20,d1,20.0,20,20,null,null", + }; + + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s2,s3,s4,s5 from table0 where device='d1' and s1 > 11.0 and s2 <= 33 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectAllAlignedWithTimeAndValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "9,d1,9.0,9,9,false,aligned_test9", + "11,d1,11.0,11,11,null,null", + "12,d1,12.0,12,12,null,null", + "14,d1,14.0,14,14,null,null", + "15,d1,15.0,15,15,null,null", + "16,d1,16.0,16,16,null,null", + "17,d1,17.0,17,17,null,null", + "18,d1,18.0,18,18,null,null", + }; + + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s2,s3,s4,s5 from table0 where device='d1' and time >= 9 and time <= 33 and s1 < 19.0 order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterAlignByDeviceTest1() { + String[] retArray = + new String[] { + "1,d1,1.0,true,aligned_test1", + "2,d1,2.0,null,aligned_test2", + "4,d1,4.0,true,aligned_test4", + "5,d1,5.0,true,aligned_test5", + "6,d1,6.0,true,null", + "7,d1,7.0,false,aligned_test7", + "8,d1,8.0,null,aligned_test8", + "9,d1,9.0,false,aligned_test9", + "11,d1,11.0,null,null", + "12,d1,12.0,null,null", + "14,d1,14.0,null,null", + "15,d1,15.0,null,null", + "16,d1,16.0,null,null", + "34,d1,null,null,aligned_test34", + }; + + String[] columnNames = {"Device", "s1", "s4", "s5"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s4,s5 from table0 where device='d1' and s1 < 17.0 or s5 = 'aligned_test34' order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectSomeAlignedWithValueFilterAlignByDeviceTest2() { + String[] retArray = + new String[] { + "7,d1,7.0,false", "9,d1,9.0,false", + }; + + String[] columnNames = {"Device", "s1", "s4"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = + statement.executeQuery( + "select Time,Device,s1,s4 from table0 where device='d1' and s1 < 19.0 and s4 = false order by time")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length + 1, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + builder.append(resultSet.getLong(1)); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(",").append(resultSet.getString(index)); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithoutTimeFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "20", "29", "28", "19", "20"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedAndNonAlignedWithoutTimeFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1,20,29,28,19,20,", "d2,19,29,28,18,19,"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from root.sg1.* align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (String columnName : columnNames) { + int index = map.get(columnName); + builder.append(resultSet.getString(index)).append(","); + } + assertEquals(retArray[cnt], builder.toString()); + cnt++; + } + assertEquals(retArray.length, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithTimeFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "12", "15", "22", "13", "6"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(*) from d1 where time >= 9 and time <= 33 align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** aggregate multi columns of aligned timeseries in one SQL */ + @Ignore + @Test + public void aggregateSomeAlignedWithoutTimeFilterAlignByDeviceTest() { + double[] retArray = + new double[] { + 20, 29, 28, 390184, 130549, 390417, 19509.2, 4501.689655172413, 13943.464285714286 + }; + String[] columnNames = { + "Device", + "count(s1)", + "count(s2)", + "count(s3)", + "sum(s1)", + "sum(s2)", + "sum(s3)", + "avg(s1)", + "avg(s2)", + "avg(s3)", + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1),count(s2),count(s3),sum(s1),sum(s2),sum(s3),avg(s1),avg(s2),avg(s3) from d1 align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + double[] ans = new double[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 1; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i - 1] = Double.parseDouble(resultSet.getString(index)); + assertEquals(retArray[i - 1], ans[i - 1], DELTA); + } + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "11"}; + String[] columnNames = {"Device", "count(s4)"}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(s4) from d1 where s4 = true align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationFuncAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = + new String[] {"d1", "8", "42.0", "5.25", "1.0", "9.0", "1", "9", "1.0", "9.0"}; + String[] columnNames = { + "Device", + "count(s1)", + "sum(s1)", + "avg(s1)", + "first_value(s1)", + "last_value(s1)", + "min_time(s1)", + "max_time(s1)", + "min_value(s1)", + "max_value(s1)", + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s1), avg(s1), " + + "first_value(s1), last_value(s1), " + + "min_time(s1), max_time(s1)," + + "max_value(s1), min_value(s1) from d1 where s1 < 10 " + + "align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countAllAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "6", "6", "9", "11", "6"}; + String[] columnNames = { + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from d1 where s4 = true " + "align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void aggregationAllAlignedWithValueFilterAlignByDeviceTest() { + String[] retArray = new String[] {"d1", "160016.0", "11", "1", "13"}; + String[] columnNames = { + "Device", "sum(s1)", "count(s4)", "min_value(s3)", "max_time(s2)", + }; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select sum(s1), count(s4), min_value(s3), max_time(s2) from d1 where s4 = true " + + "align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + Map map = new HashMap<>(); // used to adjust result sequence + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + map.put(resultSetMetaData.getColumnName(i), i); + } + assertEquals(columnNames.length, resultSetMetaData.getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + String[] ans = new String[columnNames.length]; + // No need to add time column for aggregation query + for (int i = 0; i < columnNames.length; i++) { + String columnName = columnNames[i]; + int index = map.get(columnName); + ans[i] = resultSet.getString(index); + } + assertArrayEquals(retArray, ans); + cnt++; + } + assertEquals(1, cnt); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Ignore + @Test + public void countSumAvgGroupByTimeAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,4,40.0,7.5", + "11,d1,10,130142.0,13014.2", + "21,d1,1,null,230000.0", + "31,d1,0,355.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueGroupByTimeAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,10,6.0,10,6", + "11,d1,130000,11.0,20,11", + "21,d1,230000,230000.0,null,21", + "31,d1,null,null,40,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(maxValue("s3")) + + "," + + resultSet.getString(minValue("s1")) + + "," + + resultSet.getString(maxTime("s2")) + + "," + + resultSet.getString(minTime("s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByTimeAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,null,null", "6,d1,true,aligned_test7", + "11,d1,true,aligned_unseq_test13", "16,d1,null,null", + "21,d1,true,null", "26,d1,false,null", + "31,d1,null,aligned_test31", "36,d1,null,aligned_test36" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(lastValue("s4")) + + "," + + resultSet.getString(firstValue("s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,9,9,8,8,9,9.0,10,10,true,aligned_test10", + "11,d1,10,10,10,1,1,20.0,20,20,true,aligned_unseq_test13", + "21,d1,1,0,10,10,0,230000.0,null,30,false,null", + "31,d1,0,10,0,0,10,null,40,null,null,aligned_test40" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(*), last_value(*) from d1 GROUP BY ([1, 41), 10ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(count("s2")) + + "," + + resultSet.getString(count("s3")) + + "," + + resultSet.getString(count("s4")) + + "," + + resultSet.getString(count("s5")) + + "," + + resultSet.getString(lastValue("s1")) + + "," + + resultSet.getString(lastValue("s2")) + + "," + + resultSet.getString(lastValue("s3")) + + "," + + resultSet.getString(lastValue("s4")) + + "," + + resultSet.getString(lastValue("s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithNonAlignedTimeseriesAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,0,null,null", + "7,d1,3,34.0,8.0", + "13,d1,4,130045.0,32511.25", + "19,d1,2,39.0,19.5", + "25,d1,0,null,null", + "31,d1,0,130.0,null", + "37,d1,0,154.0,null", + "1,d2,0,null,null", + "7,d2,3,34.0,8.0", + "13,d2,4,58.0,14.5", + "19,d2,2,39.0,19.5", + "25,d2,0,null,null", + "31,d2,0,130.0,null", + "37,d2,0,154.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from root.sg1.* " + + "where time > 5 GROUP BY ([1, 41), 4ms, 6ms) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgPreviousFillAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,4,40.0,7.5", + "11,d1,10,130142.0,13014.2", + "21,d1,1,130142.0,230000.0", + "31,d1,0,355.0,230000.0" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms) FILL (previous) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgValueFillAlignByDeviceTest() throws SQLException { + String[] retArray = + new String[] { + "1,d1,1,3.0,30000.0", + "6,d1,4,40.0,7.5", + "11,d1,5,130052.0,26010.4", + "16,d1,5,90.0,18.0", + "21,d1,1,3.0,230000.0", + "26,d1,0,3.0,3.0", + "31,d1,0,3.0,3.0" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where s3 > 5 and time < 30 GROUP BY ([1, 36), 5ms) FILL (3) align by device")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(ColumnHeaderConstant.DEVICE) + + "," + + resultSet.getString(count("s1")) + + "," + + resultSet.getString(sum("s2")) + + "," + + resultSet.getString(avg("s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } + } + + // --------------------------GroupByWithoutValueFilter-------------------------- + @Ignore + @Test + public void countSumAvgGroupByTest1() throws SQLException { + String[] retArray = + new String[] { + "1,4,40.0,7.5", "11,10,130142.0,13014.2", "21,1,null,230000.0", "31,0,355.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + " where time > 5 GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgGroupByTest2() throws SQLException { + String[] retArray = + new String[] { + "1,0,null,null", "6,4,40.0,7.5", "11,5,130052.0,26010.4", "16,5,90.0,18.0", + "21,1,null,230000.0", "26,0,null,null", "31,0,165.0,null", "36,0,73.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgGroupByWithSlidingStepTest() throws SQLException { + String[] retArray = + new String[] { + "1,0,null,null", + "7,3,34.0,8.0", + "13,4,130045.0,32511.25", + "19,2,39.0,19.5", + "25,0,null,null", + "31,0,130.0,null", + "37,0,154.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + " where time > 5 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d1.s1")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void countSumAvgGroupByWithNonAlignedTimeseriesTest() throws SQLException { + String[] retArray = + new String[] { + "1,0,null,null,0,null,null", + "7,3,34.0,8.0,4,34.0,8.5", + "13,4,58.0,14.5,4,130045.0,14.5", + "19,2,39.0,19.5,4,39.0,20.5", + "25,0,null,null,4,null,26.5", + "31,0,130.0,null,0,130.0,null", + "37,0,154.0,null,0,154.0,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(d1.s1), sum(d2.s2), avg(d2.s1), count(d1.s3), sum(d1.s2), avg(d2.s3) " + + "from root.sg1 where time > 5 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d2.s2")) + + "," + + resultSet.getString(avg("d2.s1")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d2.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(d1.s1), sum(d2.s2), avg(d2.s1), count(d1.s3), sum(d1.s2), avg(d2.s3) " + + "from root.sg1 where time > 5 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(sum("d2.s2")) + + "," + + resultSet.getString(avg("d2.s1")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(sum("d1.s2")) + + "," + + resultSet.getString(avg("d2.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByTest1() throws SQLException { + String[] retArray = + new String[] { + "1,10,6.0,10,6", + "11,130000,11.0,20,11", + "21,230000,230000.0,null,21", + "31,null,null,40,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + " where time > 5 GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByTest2() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null", + "6,10,6.0,10,6", + "11,130000,11.0,15,11", + "16,20,16.0,20,16", + "21,230000,230000.0,null,21", + "26,30,null,null,26", + "31,null,null,35,null", + "36,null,null,37,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByWithSlidingStepTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null", + "7,10,7.0,10,7", + "13,130000,14.0,16,13", + "19,22,19.0,20,19", + "25,28,null,null,25", + "31,null,null,34,null", + "37,null,null,40,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + "where time > 5 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(s3), min_value(s1), max_time(s2), min_time(s3) from d1 " + + " where time > 5 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d1.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void maxMinValueTimeGroupByWithNonAlignedTimeseriesTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null,null,null,null,null", + "7,10,7.0,10,7,10,7.0,10,7", + "13,16,14.0,16,13,130000,13.0,16,13", + "19,22,19.0,20,19,22,19.0,20,19", + "25,28,null,null,25,28,null,null,25", + "31,null,null,34,null,null,null,34,null", + "37,null,null,37,null,null,null,37,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(d2.s3), min_value(d1.s1), max_time(d2.s2), min_time(d1.s3), " + + "max_value(d1.s3), min_value(d2.s1), max_time(d1.s2), min_time(d2.s3) " + + "from root.sg1 where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d2.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d2.s2")) + + "," + + resultSet.getString(minTime("d1.s3")) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d2.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d2.s3")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select max_value(d2.s3), min_value(d1.s1), max_time(d2.s2), min_time(d1.s3), " + + "max_value(d1.s3), min_value(d2.s1), max_time(d1.s2), min_time(d2.s3) " + + "from root.sg1 where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(maxValue("d2.s3")) + + "," + + resultSet.getString(minValue("d1.s1")) + + "," + + resultSet.getString(maxTime("d2.s2")) + + "," + + resultSet.getString(minTime("d1.s3")) + + "," + + resultSet.getString(maxValue("d1.s3")) + + "," + + resultSet.getString(minValue("d2.s1")) + + "," + + resultSet.getString(maxTime("d1.s2")) + + "," + + resultSet.getString(minTime("d2.s3")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByTest1() throws SQLException { + String[] retArray = + new String[] { + "1,true,aligned_test7", + "11,true,aligned_unseq_test13", + "21,false,null", + "31,null,aligned_test31" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + " where time > 5 GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByTest2() throws SQLException { + String[] retArray = + new String[] { + "1,null,null", "6,true,aligned_test7", "11,true,aligned_unseq_test13", "16,null,null", + "21,true,null", "26,false,null", "31,null,aligned_test31", "36,null,aligned_test36" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 5ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 5ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByWithSlidingStepTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null", + "7,true,aligned_test7", + "13,true,aligned_unseq_test13", + "19,true,null", + "25,false,null", + "31,null,aligned_test31", + "37,null,aligned_test37" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s4), first_value(s5) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(firstValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void firstLastGroupByWithNonAlignedTimeseriesTest() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null", + "7,non_aligned_test10,false,aligned_test10,false", + "13,null,true,aligned_unseq_test13,null", + "19,null,true,null,true", + "25,null,true,null,true", + "31,non_aligned_test34,null,aligned_test34,null", + "37,non_aligned_test37,null,aligned_test37,null" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(d2.s5), first_value(d1.s4), last_value(d1.s5), first_value(d2.s4) " + + "from root.sg1 where time > 5 and time < 38 GROUP BY ([1, 41), 4ms, 6ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d2.s5")) + + "," + + resultSet.getString(firstValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")) + + "," + + resultSet.getString(firstValue("d2.s4")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(d2.s5), first_value(d1.s4), last_value(d1.s5), first_value(d2.s4) " + + "from root.sg1 where time > 5 and time < 38 " + + "GROUP BY ([1, 41), 4ms, 6ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d2.s5")) + + "," + + resultSet.getString(firstValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")) + + "," + + resultSet.getString(firstValue("d2.s4")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardTest1() throws SQLException { + String[] retArray = + new String[] { + "1,9,9,8,8,9,9.0,10,10,true,aligned_test10", + "11,10,10,10,1,1,20.0,20,20,true,aligned_unseq_test13", + "21,1,0,10,10,0,230000.0,null,30,false,null", + "31,0,10,0,0,10,null,40,null,null,aligned_test40" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(*), last_value(*) from d1 GROUP BY ([1, 41), 10ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(*), last_value(*) from d1 " + + "GROUP BY ([1, 41), 10ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardTest2() throws SQLException { + String[] retArray = + new String[] { + "1,0,0,0,0,0", + "5,2,2,2,2,1", + "9,2,3,3,2,2", + "13,3,3,3,1,1", + "17,3,3,3,0,0", + "21,1,0,3,3,0", + "25,0,0,3,3,0", + "29,0,1,2,2,1", + "33,0,3,0,0,3", + "37,0,1,0,0,1" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select count(*) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select count(*) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(count("d1.s1")) + + "," + + resultSet.getString(count("d1.s2")) + + "," + + resultSet.getString(count("d1.s3")) + + "," + + resultSet.getString(count("d1.s4")) + + "," + + resultSet.getString(count("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithWildcardTest3() throws SQLException { + String[] retArray = + new String[] { + "1,null,null,null,null,null", + "5,7.0,7,7,false,aligned_test7", + "9,11.0,11,11,true,aligned_test10", + "13,15.0,15,15,true,aligned_unseq_test13", + "17,19.0,19,19,null,null", + "21,230000.0,null,230000,false,null", + "25,null,null,27,false,null", + "29,null,31,30,false,aligned_test31", + "33,null,35,null,null,aligned_test35", + "37,null,37,null,null,aligned_test37" + }; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + int cnt; + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(*) from d1 " + + "where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms)")) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(*) from d1 " + + " where time > 5 and time < 38 GROUP BY ([1, 41), 3ms, 4ms) order by time desc")) { + cnt = retArray.length; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + + "," + + resultSet.getString(lastValue("d1.s1")) + + "," + + resultSet.getString(lastValue("d1.s2")) + + "," + + resultSet.getString(lastValue("d1.s3")) + + "," + + resultSet.getString(lastValue("d1.s4")) + + "," + + resultSet.getString(lastValue("d1.s5")); + Assert.assertEquals(retArray[cnt - 1], ans); + cnt--; + } + Assert.assertEquals(0, cnt); + } + } + } + + @Ignore + @Test + public void groupByWithoutAggregationFuncTest() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + statement.executeQuery("select s1 from d1 group by ([0, 100), 5ms)"); + + fail("No expected exception thrown"); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage(), + e.getMessage() + .contains( + "Common queries and aggregated queries are not allowed to appear at the same time")); + } + } + + @Ignore + @Test + public void negativeOrZeroTimeIntervalTest() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + statement.executeQuery( + "select count(s1), sum(s2), avg(s1) from d1 " + + "where time > 5 GROUP BY ([1, 41), 0ms)"); + fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage(), + e.getMessage() + .contains("The second parameter time interval should be a positive integer.")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView2IT.java new file mode 100644 index 0000000000000..072347606809c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView2IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTableView2IT extends IoTDBPredicatePushDownTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(2); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView3IT.java new file mode 100644 index 0000000000000..e7609acb0ede8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView3IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTableView3IT extends IoTDBPredicatePushDownTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setMaxNumberOfPointsInPage(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView4IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView4IT.java new file mode 100644 index 0000000000000..3cb7dc3422309 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableView4IT.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTableView4IT extends IoTDBPredicatePushDownTableViewIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3) + .setDegreeOfParallelism(4); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableViewIT.java new file mode 100644 index 0000000000000..7a7a538c7a175 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/IoTDBPredicatePushDownTableViewIT.java @@ -0,0 +1,744 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPredicatePushDownTableViewIT { + + private final String database = "db"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setMaxTsBlockLineNumber(3); + EnvFactory.getEnv().initClusterEnvironment(); + TableViewUtils.insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAlignedRawDataAlignByTime1() { + String[] expectedHeader1 = new String[] {"Time", "s2", "s3"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.010Z,10,10,", + "1970-01-01T00:00:00.011Z,11,11,", + "1970-01-01T00:00:00.012Z,12,12,", + "1970-01-01T00:00:00.014Z,14,14,", + "1970-01-01T00:00:00.015Z,15,15,", + "1970-01-01T00:00:00.016Z,16,16,", + "1970-01-01T00:00:00.017Z,17,17,", + "1970-01-01T00:00:00.018Z,18,18,", + "1970-01-01T00:00:00.019Z,19,19,", + "1970-01-01T00:00:00.020Z,20,20," + }; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s3"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s2"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s2"}; + String[] retArray4 = + new String[] {"1970-01-01T00:00:00.014Z,14,", "1970-01-01T00:00:00.015Z,15,"}; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d1' and s2 - 1 >= 9 and s2 < 30 offset 3 limit 2", + expectedHeader4, + retArray4, + database); + } + + @Test + public void testAlignedRawDataAlignByTime2() { + String[] expectedHeader1 = new String[] {"Time", "s2", "_col2"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.003Z,null,30001,", + "1970-01-01T00:00:00.013Z,130000,130001,", + "1970-01-01T00:00:00.016Z,16,17,", + "1970-01-01T00:00:00.017Z,17,18,", + "1970-01-01T00:00:00.018Z,18,19,", + "1970-01-01T00:00:00.019Z,19,20,", + "1970-01-01T00:00:00.020Z,20,21,", + "1970-01-01T00:00:00.021Z,null,22,", + "1970-01-01T00:00:00.022Z,null,23,", + "1970-01-01T00:00:00.023Z,null,230001,", + "1970-01-01T00:00:00.024Z,null,25,", + "1970-01-01T00:00:00.025Z,null,26,", + "1970-01-01T00:00:00.026Z,null,27,", + "1970-01-01T00:00:00.027Z,null,28,", + "1970-01-01T00:00:00.028Z,null,29,", + "1970-01-01T00:00:00.029Z,null,30,", + "1970-01-01T00:00:00.030Z,null,31,", + }; + tableResultSetEqualTest( + "select Time,s2, s3 + 1 from table0 where device='d1' and s3 + 1 > 16", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s2"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.003Z,null,", + "1970-01-01T00:00:00.013Z,130000,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,null,", + "1970-01-01T00:00:00.022Z,null,", + "1970-01-01T00:00:00.023Z,null,", + "1970-01-01T00:00:00.024Z,null,", + "1970-01-01T00:00:00.025Z,null,", + "1970-01-01T00:00:00.026Z,null,", + "1970-01-01T00:00:00.027Z,null,", + "1970-01-01T00:00:00.028Z,null,", + "1970-01-01T00:00:00.029Z,null,", + "1970-01-01T00:00:00.030Z,null,", + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d1' and s3 + 1 > 16", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s3"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.003Z,30000,", + "1970-01-01T00:00:00.013Z,130000,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,21,", + "1970-01-01T00:00:00.022Z,22,", + "1970-01-01T00:00:00.023Z,230000,", + "1970-01-01T00:00:00.024Z,24,", + "1970-01-01T00:00:00.025Z,25,", + "1970-01-01T00:00:00.026Z,26,", + "1970-01-01T00:00:00.027Z,27,", + "1970-01-01T00:00:00.028Z,28,", + "1970-01-01T00:00:00.029Z,29,", + "1970-01-01T00:00:00.030Z,30,", + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d1' and s3 + 1 > 16", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s3"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.003Z,30000,", + "1970-01-01T00:00:00.013Z,130000,", + "1970-01-01T00:00:00.016Z,16," + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d1' and s3 + 1 > 16 limit 3", + expectedHeader4, + retArray4, + database); + } + + @Test + public void testNonAlignedRawDataAlignByTime1() { + String[] expectedHeader1 = new String[] {"Time", "s2", "s3"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.010Z,10,10,", + "1970-01-01T00:00:00.011Z,11,11,", + "1970-01-01T00:00:00.012Z,12,12,", + "1970-01-01T00:00:00.013Z,13,13,", + "1970-01-01T00:00:00.014Z,14,14,", + "1970-01-01T00:00:00.015Z,15,15,", + "1970-01-01T00:00:00.016Z,16,16,", + "1970-01-01T00:00:00.017Z,17,17,", + "1970-01-01T00:00:00.018Z,18,18,", + "1970-01-01T00:00:00.019Z,19,19,", + "1970-01-01T00:00:00.020Z,20,20," + }; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s3"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.013Z,13,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s2"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.010Z,10,", + "1970-01-01T00:00:00.011Z,11,", + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.013Z,13,", + "1970-01-01T00:00:00.014Z,14,", + "1970-01-01T00:00:00.015Z,15,", + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20," + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s2"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.012Z,12,", + "1970-01-01T00:00:00.013Z,13,", + "1970-01-01T00:00:00.014Z,14," + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d2' and s2 - 1 >= 9 and s2 < 30 offset 2 limit 3", + expectedHeader4, + retArray4, + database); + } + + @Test + public void testNonAlignedRawDataAlignByTime2() { + String[] expectedHeader1 = new String[] {"Time", "s2", "s3"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.016Z,16,16,", + "1970-01-01T00:00:00.017Z,17,17,", + "1970-01-01T00:00:00.018Z,18,18,", + "1970-01-01T00:00:00.019Z,19,19,", + "1970-01-01T00:00:00.020Z,20,20,", + "1970-01-01T00:00:00.021Z,null,21,", + "1970-01-01T00:00:00.022Z,null,22,", + "1970-01-01T00:00:00.023Z,null,23,", + "1970-01-01T00:00:00.024Z,null,24,", + "1970-01-01T00:00:00.025Z,null,25,", + "1970-01-01T00:00:00.026Z,null,26,", + "1970-01-01T00:00:00.027Z,null,27,", + "1970-01-01T00:00:00.028Z,null,28,", + "1970-01-01T00:00:00.029Z,null,29,", + "1970-01-01T00:00:00.030Z,null,30,", + }; + tableResultSetEqualTest( + "select Time,s2, s3 from table0 where device='d2' and s3 + 1 > 16", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"Time", "s2"}; + String[] retArray2 = + new String[] { + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,null,", + "1970-01-01T00:00:00.022Z,null,", + "1970-01-01T00:00:00.023Z,null,", + "1970-01-01T00:00:00.024Z,null,", + "1970-01-01T00:00:00.025Z,null,", + "1970-01-01T00:00:00.026Z,null,", + "1970-01-01T00:00:00.027Z,null,", + "1970-01-01T00:00:00.028Z,null,", + "1970-01-01T00:00:00.029Z,null,", + "1970-01-01T00:00:00.030Z,null,", + }; + tableResultSetEqualTest( + "select Time,s2 from table0 where device='d2' and s3 + 1 > 16", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"Time", "s3"}; + String[] retArray3 = + new String[] { + "1970-01-01T00:00:00.016Z,16,", + "1970-01-01T00:00:00.017Z,17,", + "1970-01-01T00:00:00.018Z,18,", + "1970-01-01T00:00:00.019Z,19,", + "1970-01-01T00:00:00.020Z,20,", + "1970-01-01T00:00:00.021Z,21,", + "1970-01-01T00:00:00.022Z,22,", + "1970-01-01T00:00:00.023Z,23,", + "1970-01-01T00:00:00.024Z,24,", + "1970-01-01T00:00:00.025Z,25,", + "1970-01-01T00:00:00.026Z,26,", + "1970-01-01T00:00:00.027Z,27,", + "1970-01-01T00:00:00.028Z,28,", + "1970-01-01T00:00:00.029Z,29,", + "1970-01-01T00:00:00.030Z,30,", + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d2' and s3 + 1 > 16", + expectedHeader3, + retArray3, + database); + + String[] expectedHeader4 = new String[] {"Time", "s3"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.026Z,26,", + "1970-01-01T00:00:00.027Z,27,", + "1970-01-01T00:00:00.028Z,28,", + "1970-01-01T00:00:00.029Z,29,", + "1970-01-01T00:00:00.030Z,30,", + }; + tableResultSetEqualTest( + "select Time,s3 from table0 where device='d2' and s3 + 1 > 16 offset 10", + expectedHeader4, + retArray4, + database); + } + + @Ignore + @Test + public void testAlignedAggregationAlignByTime1() { + String[] expectedHeader1 = new String[] {"count(d1.s2),count(d1.s3),"}; + String[] retArray1 = + new String[] { + "10,10,", + }; + tableResultSetEqualTest( + "select count(s2), count(s3) from d1 where s2 - 1 >= 9 and s2 < 30", + expectedHeader1, + retArray1, + database); + + String[] expectedHeader2 = new String[] {"count(d1.s3),"}; + String[] retArray2 = new String[] {"10,"}; + tableResultSetEqualTest( + "select count(s3) from d1 where s2 - 1 >= 9 and s2 < 30", + expectedHeader2, + retArray2, + database); + + String[] expectedHeader3 = new String[] {"count(d1.s2),"}; + String[] retArray3 = + new String[] { + "10,", + }; + tableResultSetEqualTest( + "select count(s2) from d1 where s2 - 1 >= 9 and s2 < 30", + expectedHeader3, + retArray3, + database); + } + + // @Ignore + // @Test + // public void testAlignedAggregationAlignByTime2() { + // String[] expectedHeader1 = new String[]{"count(d1.s2),count(d1.s3 + 1),"}; + // String[] retArray1 = + // new String[] { + // "6,17,", + // }; + // tableResultSetEqualTest( + // "select count(s2), count(s3 + 1) from d1 where s3 + 1 > 16", + // expectedHeader1, + // retArray1, database); + // + // String[] expectedHeader2 = "count(d1.s2),"; + // String[] retArray2 = + // new String[] { + // "6,", + // }; + // tableResultSetEqualTest( + // "select count(s2) from d1 where s3 + 1 > 16", expectedHeader2, retArray2, database); + // + // String[] expectedHeader3 = "count(d1.s3),"; + // String[] retArray3 = + // new String[] { + // "17,", + // }; + // tableResultSetEqualTest( + // "select count(s3) from d1 where s3 + 1 > 16", expectedHeader3, retArray3, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedAggregationAlignByTime1() { + // String[] expectedHeader1 = "count(d2.s2),count(d2.s3),"; + // String[] retArray1 = new String[] {"11,11,"}; + // tableResultSetEqualTest( + // "select count(s2), count(s3) from d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader1, + // retArray1, database); + // + // String[] expectedHeader2 = "count(d2.s3),"; + // String[] retArray2 = new String[] {"11,"}; + // tableResultSetEqualTest( + // "select count(s3) from d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader2, + // retArray2, database); + // + // String[] expectedHeader3 = "count(d2.s2),"; + // String[] retArray3 = new String[] {"11,"}; + // tableResultSetEqualTest( + // "select count(s2) from d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader3, + // retArray3, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedAggregationAlignByTime2() { + // String[] expectedHeader1 = "count(d2.s2),count(d2.s3),"; + // String[] retArray1 = + // new String[] { + // "5,15,", + // }; + // tableResultSetEqualTest( + // "select count(s2), count(s3) from d2 where s3 + 1 > 16", + // expectedHeader1, + // retArray1, database); + // + // String[] expectedHeader2 = "count(d2.s2),"; + // String[] retArray2 = + // new String[] { + // "5,", + // }; + // tableResultSetEqualTest( + // "select count(s2) from d2 where s3 + 1 > 16", expectedHeader2, retArray2, database); + // + // String[] expectedHeader3 = "count(d2.s3),"; + // String[] retArray3 = + // new String[] { + // "15,", + // }; + // tableResultSetEqualTest( + // "select count(s3) from d2 where s3 + 1 > 16", expectedHeader3, retArray3, database); + // } + // + // @Ignore + // @Test + // public void testMixAggregationAlignByTime() { + // String[] expectedHeader1 = + // "count(d1.s2),count(d2.s2),count(d1.s3),count(d2.s3),"; + // String[] retArray1 = + // new String[] { + // "10,10,10,10,", + // }; + // tableResultSetEqualTest( + // "select count(s2), count(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30", + // expectedHeader1, + // retArray1, database); + // } + // + // @Ignore + // @Test + // public void testAlignedGroupByTimeAlignByTime1() { + // String[] expectedHeader = "Time,count(d1.s2),sum(d1.s3),"; + // String[] retArray = new String[] {"1,1,10.0,", "11,9,142.0,", "21,0,null,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d1 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), + // 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testAlignedGroupByTimeAlignByTime2() { + // String[] expectedHeader = "Time,count(d1.s2),sum(d1.s3),"; + // String[] retArray = + // new String[] {"1,0,30000.0,", "11,6,130090.0,", "21,0,230232.0,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d1 where s3 + 1 > 16 group by ([1, 41), 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedGroupByTimeAlignByTime1() { + // String[] expectedHeader = "Time,count(d2.s2),sum(d2.s3 + 1),"; + // String[] retArray = new String[] {"1,1,11.0,", "11,10,165.0,", "21,0,null,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3 + 1) from d2 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), + // 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testNonAlignedGroupByTimeAlignByTime2() { + // String[] expectedHeader = "Time,count(d2.s2),sum(d2.s3),"; + // String[] retArray = new String[] {"1,0,null,", "11,5,90.0,", "21,0,255.0,", "31,0,null,"}; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d2 where s3 + 1 > 16 group by ([1, 41), 10ms)", + // expectedHeader, + // retArray, database); + // } + // + // @Ignore + // @Test + // public void testMixGroupByTimeAlignByTime() { + // String[] expectedHeader = + // "Time,count(d1.s2),count(d2.s2),sum(d1.s3),sum(d2.s3),"; + // String[] retArray = + // new String[] { + // "1,1,1,10.0,10.0,", "11,9,9,142.0,142.0,", "21,0,0,null,null,", "31,0,0,null,null," + // }; + // tableResultSetEqualTest( + // "select count(s2), sum(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), + // 10ms)", + // expectedHeader, + // retArray, database); + // } + + @Test + public void testRawDataAlignByDevice1() { + String[] expectedHeader = new String[] {"Time", "Device", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.010Z,d2,10,10,", + "1970-01-01T00:00:00.011Z,d2,11,11,", + "1970-01-01T00:00:00.012Z,d2,12,12,", + "1970-01-01T00:00:00.013Z,d2,13,13,", + "1970-01-01T00:00:00.014Z,d2,14,14,", + "1970-01-01T00:00:00.015Z,d2,15,15,", + "1970-01-01T00:00:00.016Z,d2,16,16,", + "1970-01-01T00:00:00.017Z,d2,17,17,", + "1970-01-01T00:00:00.018Z,d2,18,18,", + "1970-01-01T00:00:00.019Z,d2,19,19,", + "1970-01-01T00:00:00.020Z,d2,20,20,", + "1970-01-01T00:00:00.010Z,d1,10,10,", + "1970-01-01T00:00:00.011Z,d1,11,11,", + "1970-01-01T00:00:00.012Z,d1,12,12,", + "1970-01-01T00:00:00.014Z,d1,14,14,", + "1970-01-01T00:00:00.015Z,d1,15,15,", + "1970-01-01T00:00:00.016Z,d1,16,16,", + "1970-01-01T00:00:00.017Z,d1,17,17,", + "1970-01-01T00:00:00.018Z,d1,18,18,", + "1970-01-01T00:00:00.019Z,d1,19,19,", + "1970-01-01T00:00:00.020Z,d1,20,20,", + }; + tableResultSetEqualTest( + "select Time, Device,s2, s3 from table0 where s2 - 1 >= 9 and s2 < 30 order by device desc", + expectedHeader, + retArray, + database); + } + + @Test + public void testRawDataAlignByDevice2() { + String[] expectedHeader = new String[] {"Time", "Device", "s2", "_col3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,30001,", + "1970-01-01T00:00:00.013Z,d1,130000,130001,", + "1970-01-01T00:00:00.016Z,d1,16,17,", + "1970-01-01T00:00:00.017Z,d1,17,18,", + "1970-01-01T00:00:00.018Z,d1,18,19,", + "1970-01-01T00:00:00.019Z,d1,19,20,", + "1970-01-01T00:00:00.020Z,d1,20,21,", + "1970-01-01T00:00:00.021Z,d1,null,22,", + "1970-01-01T00:00:00.022Z,d1,null,23,", + "1970-01-01T00:00:00.023Z,d1,null,230001,", + "1970-01-01T00:00:00.024Z,d1,null,25,", + "1970-01-01T00:00:00.025Z,d1,null,26,", + "1970-01-01T00:00:00.026Z,d1,null,27,", + "1970-01-01T00:00:00.027Z,d1,null,28,", + "1970-01-01T00:00:00.028Z,d1,null,29,", + "1970-01-01T00:00:00.029Z,d1,null,30,", + "1970-01-01T00:00:00.030Z,d1,null,31,", + "1970-01-01T00:00:00.016Z,d2,16,17,", + "1970-01-01T00:00:00.017Z,d2,17,18,", + "1970-01-01T00:00:00.018Z,d2,18,19,", + "1970-01-01T00:00:00.019Z,d2,19,20,", + "1970-01-01T00:00:00.020Z,d2,20,21,", + "1970-01-01T00:00:00.021Z,d2,null,22,", + "1970-01-01T00:00:00.022Z,d2,null,23,", + "1970-01-01T00:00:00.023Z,d2,null,24,", + "1970-01-01T00:00:00.024Z,d2,null,25,", + "1970-01-01T00:00:00.025Z,d2,null,26,", + "1970-01-01T00:00:00.026Z,d2,null,27,", + "1970-01-01T00:00:00.027Z,d2,null,28,", + "1970-01-01T00:00:00.028Z,d2,null,29,", + "1970-01-01T00:00:00.029Z,d2,null,30,", + "1970-01-01T00:00:00.030Z,d2,null,31,", + }; + tableResultSetEqualTest( + "select Time,Device,s2, s3 + 1 from table0 where s3 + 1 > 16 order by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testAggregationAlignByDevice1() { + String[] expectedHeader = new String[] {"Device,count(s2),sum(s3),"}; + String[] retArray = new String[] {"d2,11,165.0,", "d1,10,152.0,"}; + tableResultSetEqualTest( + "select count(s2), sum(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30 order by device desc align by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testAggregationAlignByDevice2() { + String[] expectedHeader = new String[] {"Device,count(s2),sum(s3 + 1),"}; + String[] retArray = new String[] {"d1,6,390339.0,", "d2,5,360.0,"}; + tableResultSetEqualTest( + "select count(s2), sum(s3 + 1) from d1, d2 where s3 + 1 > 16 align by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testGroupByTimeAlignByDevice1() { + String[] expectedHeader = new String[] {"Time,Device,count(s2),sum(s3),"}; + String[] retArray = + new String[] { + "1,d2,1,10.0,", + "11,d2,10,155.0,", + "21,d2,0,null,", + "31,d2,0,null,", + "1,d1,1,10.0,", + "11,d1,9,142.0,", + "21,d1,0,null,", + "31,d1,0,null," + }; + tableResultSetEqualTest( + "select count(s2), sum(s3) from d1, d2 where s2 - 1 >= 9 and s2 < 30 group by ([1, 41), 10ms) order by device desc align by device", + expectedHeader, + retArray, + database); + } + + @Ignore + @Test + public void testGroupByTimeAlignByDevice2() { + String[] expectedHeader = new String[] {"Time,Device,count(s2),sum(s3 + 1),"}; + String[] retArray = + new String[] { + "1,d1,0,30001.0,", + "11,d1,6,130096.0,", + "21,d1,0,230242.0,", + "31,d1,0,null,", + "1,d2,0,null,", + "11,d2,5,95.0,", + "21,d2,0,265.0,", + "31,d2,0,null," + }; + tableResultSetEqualTest( + "select count(s2), sum(s3 + 1) from d1, d2 where s3 + 1 > 16 group by ([1, 41), 10ms) align by device", + expectedHeader, + retArray, + database); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/TableViewUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/TableViewUtils.java new file mode 100644 index 0000000000000..656f1b24fafea --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/aligned/TableViewUtils.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.view.old.aligned; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * This class generates data for test cases in aligned time series scenarios. + * + *

You can comprehensively view the generated data in the following online doc: + * + *

https://docs.google.com/spreadsheets/d/1kfrSR1_paSd9B1Z0jnPBD3WQIMDslDuNm4R0mpWx9Ms/edit?usp=sharing + */ +public class TableViewUtils { + + public static final String USE_DB = "use db"; + + private static final String[] sqls = + new String[] { + "CREATE DATABASE root.sg1", + "create aligned timeseries root.sg1.d1(s1 FLOAT encoding=RLE, s2 INT32 encoding=Gorilla compression=SNAPPY, s3 INT64, s4 BOOLEAN, s5 TEXT)", + "create timeseries root.sg1.d2.s1 WITH DATATYPE=FLOAT, encoding=RLE", + "create timeseries root.sg1.d2.s2 WITH DATATYPE=INT32, encoding=Gorilla", + "create timeseries root.sg1.d2.s3 WITH DATATYPE=INT64", + "create timeseries root.sg1.d2.s4 WITH DATATYPE=BOOLEAN", + "create timeseries root.sg1.d2.s5 WITH DATATYPE=TEXT", + "insert into root.sg1.d1(time, s1, s2, s3, s4, s5) aligned values(1, 1.0, 1, 1, TRUE, 'aligned_test1')", + "insert into root.sg1.d1(time, s1, s2, s3, s5) aligned values(2, 2.0, 2, 2, 'aligned_test2')", + "insert into root.sg1.d1(time, s1, s3, s4, s5) aligned values(3, 3.0, 3, FALSE, 'aligned_test3')", + "insert into root.sg1.d1(time, s1, s2, s4, s5) aligned values(4, 4.0, 4, TRUE, 'aligned_test4')", + "insert into root.sg1.d1(time, s1, s2, s4, s5) aligned values(5, 5.0, 5, TRUE, 'aligned_test5')", + "insert into root.sg1.d1(time, s1, s2, s3, s4) aligned values(6, 6.0, 6, 6, TRUE)", + "insert into root.sg1.d1(time, s1, s2, s3, s4, s5) aligned values(7, 7.0, 7, 7, FALSE, 'aligned_test7')", + "insert into root.sg1.d1(time, s1, s2, s3, s5) aligned values(8, 8.0, 8, 8, 'aligned_test8')", + "insert into root.sg1.d1(time, s1, s2, s3, s4, s5) aligned values(9, 9.0, 9, 9, FALSE, 'aligned_test9')", + "insert into root.sg1.d1(time, s2, s3, s4, s5) aligned values(10, 10, 10, TRUE, 'aligned_test10')", + "insert into root.sg1.d2(time, s1, s2, s3, s4, s5) values(1, 1.0, 1, 1, TRUE, 'non_aligned_test1')", + "insert into root.sg1.d2(time, s1, s2, s3, s5) values(2, 2.0, 2, 2, 'non_aligned_test2')", + "insert into root.sg1.d2(time, s1, s3, s4, s5) values(3, 3.0, 3, FALSE, 'non_aligned_test3')", + "insert into root.sg1.d2(time, s1, s2, s4, s5) values(4, 4.0, 4, TRUE, 'non_aligned_test4')", + "insert into root.sg1.d2(time, s1, s2, s4, s5) values(5, 5.0, 5, TRUE, 'non_aligned_test5')", + "insert into root.sg1.d2(time, s1, s2, s3, s4) values(6, 6.0, 6, 6, TRUE)", + "insert into root.sg1.d2(time, s1, s2, s3, s4, s5) values(7, 7.0, 7, 7, FALSE, 'non_aligned_test7')", + "insert into root.sg1.d2(time, s1, s2, s3, s5) values(8, 8.0, 8, 8, 'non_aligned_test8')", + "insert into root.sg1.d2(time, s1, s2, s3, s4, s5) values(9, 9.0, 9, 9, FALSE, 'non_aligned_test9')", + "insert into root.sg1.d2(time, s2, s3, s4, s5) values(10, 10, 10, TRUE, 'non_aligned_test10')", + "flush", + "insert into root.sg1.d1(time, s1, s3, s4, s5) aligned values(3, 30000.0, 30000, TRUE, 'aligned_unseq_test3')", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(11, 11.0, 11, 11)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(12, 12.0, 12, 12)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(13, 13.0, 13, 13)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(14, 14.0, 14, 14)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(15, 15.0, 15, 15)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(16, 16.0, 16, 16)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(17, 17.0, 17, 17)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(18, 18.0, 18, 18)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(19, 19.0, 19, 19)", + "insert into root.sg1.d1(time, s1, s2, s3) aligned values(20, 20.0, 20, 20)", + "insert into root.sg1.d2(time, s1, s2, s3) values(11, 11.0, 11, 11)", + "insert into root.sg1.d2(time, s1, s2, s3) values(12, 12.0, 12, 12)", + "insert into root.sg1.d2(time, s1, s2, s3) values(13, 13.0, 13, 13)", + "insert into root.sg1.d2(time, s1, s2, s3) values(14, 14.0, 14, 14)", + "insert into root.sg1.d2(time, s1, s2, s3) values(15, 15.0, 15, 15)", + "insert into root.sg1.d2(time, s1, s2, s3) values(16, 16.0, 16, 16)", + "insert into root.sg1.d2(time, s1, s2, s3) values(17, 17.0, 17, 17)", + "insert into root.sg1.d2(time, s1, s2, s3) values(18, 18.0, 18, 18)", + "insert into root.sg1.d2(time, s1, s2, s3) values(19, 19.0, 19, 19)", + "insert into root.sg1.d2(time, s1, s2, s3) values(20, 20.0, 20, 20)", + "flush", + "insert into root.sg1.d1(time, s1, s2, s3, s4, s5) aligned values(13, 130000.0, 130000, 130000, TRUE, 'aligned_unseq_test13')", + "insert into root.sg1.d1(time, s3, s4) aligned values(21, 21, TRUE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(22, 22, TRUE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(23, 23, TRUE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(24, 24, TRUE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(25, 25, TRUE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(26, 26, FALSE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(27, 27, FALSE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(28, 28, FALSE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(29, 29, FALSE)", + "insert into root.sg1.d1(time, s3, s4) aligned values(30, 30, FALSE)", + "insert into root.sg1.d2(time, s3, s4) values(21, 21, TRUE)", + "insert into root.sg1.d2(time, s3, s4) values(22, 22, TRUE)", + "insert into root.sg1.d2(time, s3, s4) values(23, 23, TRUE)", + "insert into root.sg1.d2(time, s3, s4) values(24, 24, TRUE)", + "insert into root.sg1.d2(time, s3, s4) values(25, 25, TRUE)", + "insert into root.sg1.d2(time, s3, s4) values(26, 26, FALSE)", + "insert into root.sg1.d2(time, s3, s4) values(27, 27, FALSE)", + "insert into root.sg1.d2(time, s3, s4) values(28, 28, FALSE)", + "insert into root.sg1.d2(time, s3, s4) values(29, 29, FALSE)", + "insert into root.sg1.d2(time, s3, s4) values(30, 30, FALSE)", + "flush", + "insert into root.sg1.d1(time, s1, s3, s4) aligned values(23, 230000.0, 230000, FALSE)", + "insert into root.sg1.d1(time, s2, s5) aligned values(31, 31, 'aligned_test31')", + "insert into root.sg1.d1(time, s2, s5) aligned values(32, 32, 'aligned_test32')", + "insert into root.sg1.d1(time, s2, s5) aligned values(33, 33, 'aligned_test33')", + "insert into root.sg1.d1(time, s2, s5) aligned values(34, 34, 'aligned_test34')", + "insert into root.sg1.d1(time, s2, s5) aligned values(35, 35, 'aligned_test35')", + "insert into root.sg1.d1(time, s2, s5) aligned values(36, 36, 'aligned_test36')", + "insert into root.sg1.d1(time, s2, s5) aligned values(37, 37, 'aligned_test37')", + "insert into root.sg1.d1(time, s2, s5) aligned values(38, 38, 'aligned_test38')", + "insert into root.sg1.d1(time, s2, s5) aligned values(39, 39, 'aligned_test39')", + "insert into root.sg1.d1(time, s2, s5) aligned values(40, 40, 'aligned_test40')", + "insert into root.sg1.d2(time, s2, s5) values(31, 31, 'non_aligned_test31')", + "insert into root.sg1.d2(time, s2, s5) values(32, 32, 'non_aligned_test32')", + "insert into root.sg1.d2(time, s2, s5) values(33, 33, 'non_aligned_test33')", + "insert into root.sg1.d2(time, s2, s5) values(34, 34, 'non_aligned_test34')", + "insert into root.sg1.d2(time, s2, s5) values(35, 35, 'non_aligned_test35')", + "insert into root.sg1.d2(time, s2, s5) values(36, 36, 'non_aligned_test36')", + "insert into root.sg1.d2(time, s2, s5) values(37, 37, 'non_aligned_test37')", + "insert into root.sg1.d2(time, s2, s5) values(38, 38, 'non_aligned_test38')", + "insert into root.sg1.d2(time, s2, s5) values(39, 39, 'non_aligned_test39')", + "insert into root.sg1.d2(time, s2, s5) values(40, 40, 'non_aligned_test40')", + }; + private static final String[] createTableViewSqls = { + "CREATE DATABASE db", + "USE db", + "CREATE VIEW table0 (device string tag, s1 FLOAT field, s2 INT32 field, s3 INT64 field, s4 BOOLEAN field, s5 TEXT field) as root.sg1.**", + }; + + public static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : createTableViewSqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableResultSetEqualTest( + String sql, String[] expectedHeader, String[] expectedRetArray, String database) { + tableResultSetEqualTest( + sql, + expectedHeader, + expectedRetArray, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + database); + } + + public static void tableResultSetEqualTest( + String sql, + String[] expectedHeader, + String[] expectedRetArray, + String userName, + String password, + String database) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + + for (int i = 1; i <= expectedHeader.length; i++) { + if (i == 1) { + builder.append(resultSet.getLong(i)).append(","); + } else { + builder.append(resultSet.getString(i)).append(","); + } + } + assertEquals(expectedRetArray[cnt], builder.toString()); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/orderby/IoTDBOrderByTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/orderby/IoTDBOrderByTableViewIT.java new file mode 100644 index 0000000000000..5f733be9b922c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/orderby/IoTDBOrderByTableViewIT.java @@ -0,0 +1,1599 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.orderby; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static org.apache.iotdb.relational.it.query.old.aligned.TableUtils.USE_DB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBOrderByTableViewIT { + + // the data can be viewed in + // https://docs.google.com/spreadsheets/d/1OWA1bKraArCwWVnuTjuhJ5yLG0PFLdD78gD6FjquepI/edit#gid=0 + private static final String[] sql = + new String[] { + "CREATE DATABASE root.sg", + "CREATE TIMESERIES root.sg.d1.num WITH DATATYPE=INT32, ENCODING=RLE", + "CREATE TIMESERIES root.sg.d1.bignum WITH DATATYPE=INT64, ENCODING=RLE", + "CREATE TIMESERIES root.sg.d1.floatnum WITH DATATYPE=DOUBLE, ENCODING=RLE, 'MAX_POINT_NUMBER'='5'", + "CREATE TIMESERIES root.sg.d1.str WITH DATATYPE=TEXT, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.bool WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(0,3,2947483648,231.2121,\"coconut\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(20,2,2147483648,434.12,\"pineapple\",TRUE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(40,1,2247483648,12.123,\"apricot\",TRUE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(80,9,2147483646,43.12,\"apple\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(100,8,2147483964,4654.231,\"papaya\",TRUE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(31536000000,6,2147483650,1231.21,\"banana\",TRUE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(31536000100,10,3147483648,231.55,\"pumelo\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(31536000500,4,2147493648,213.1,\"peach\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(31536001000,5,2149783648,56.32,\"orange\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(31536010000,7,2147983648,213.112,\"lemon\",TRUE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(31536100000,11,2147468648,54.121,\"pitaya\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(41536000000,12,2146483648,45.231,\"strawberry\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(41536000020,14,2907483648,231.34,\"cherry\",FALSE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(41536900000,13,2107483648,54.12,\"lychee\",TRUE)", + "insert into root.sg.d1(timestamp,num,bignum,floatnum,str,bool) values(51536000000,15,3147483648,235.213,\"watermelon\",TRUE)" + }; + + private static final String[] createTableViewSql = { + "CREATE DATABASE db", + "USE db", + "CREATE VIEW table0 (device string tag, num int32 field, bignum int64 field, " + + "floatnum double field, str TEXT field, bool BOOLEAN field) as root.sg.**", + }; + + private static final String[] sql2 = + new String[] { + "CREATE TIMESERIES root.sg.d2.num WITH DATATYPE=INT32, ENCODING=RLE", + "CREATE TIMESERIES root.sg.d2.bignum WITH DATATYPE=INT64, ENCODING=RLE", + "CREATE TIMESERIES root.sg.d2.floatnum WITH DATATYPE=DOUBLE, ENCODING=RLE, 'MAX_POINT_NUMBER'='5'", + "CREATE TIMESERIES root.sg.d2.str WITH DATATYPE=TEXT, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d2.bool WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(0,3,2947483648,231.2121,\"coconut\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(20,2,2147483648,434.12,\"pineapple\",TRUE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(40,1,2247483648,12.123,\"apricot\",TRUE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(80,9,2147483646,43.12,\"apple\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(100,8,2147483964,4654.231,\"papaya\",TRUE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(31536000000,6,2147483650,1231.21,\"banana\",TRUE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(31536000100,10,3147483648,231.55,\"pumelo\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(31536000500,4,2147493648,213.1,\"peach\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(31536001000,5,2149783648,56.32,\"orange\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(31536010000,7,2147983648,213.112,\"lemon\",TRUE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(31536100000,11,2147468648,54.121,\"pitaya\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(41536000000,12,2146483648,45.231,\"strawberry\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(41536000020,14,2907483648,231.34,\"cherry\",FALSE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(41536900000,13,2107483648,54.12,\"lychee\",TRUE)", + "insert into root.sg.d2(timestamp,num,bignum,floatnum,str,bool) values(51536000000,15,3147483648,235.213,\"watermelon\",TRUE)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sql) { + statement.execute(sql); + } + for (String sql : sql2) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : createTableViewSql) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + // sessionInsertData1(); + // sessionInsertData2(); + } + + // ordinal data + public static final String[][] RES = + new String[][] { + {"0", "3", "2947483648", "231.2121", "coconut", "false"}, + {"20", "2", "2147483648", "434.12", "pineapple", "true"}, + {"40", "1", "2247483648", "12.123", "apricot", "true"}, + {"80", "9", "2147483646", "43.12", "apple", "false"}, + {"100", "8", "2147483964", "4654.231", "papaya", "true"}, + {"31536000000", "6", "2147483650", "1231.21", "banana", "true"}, + {"31536000100", "10", "3147483648", "231.55", "pumelo", "false"}, + {"31536000500", "4", "2147493648", "213.1", "peach", "false"}, + {"31536001000", "5", "2149783648", "56.32", "orange", "false"}, + {"31536010000", "7", "2147983648", "213.112", "lemon", "true"}, + {"31536100000", "11", "2147468648", "54.121", "pitaya", "false"}, + {"41536000000", "12", "2146483648", "45.231", "strawberry", "false"}, + {"41536000020", "14", "2907483648", "231.34", "cherry", "false"}, + {"41536900000", "13", "2107483648", "54.12", "lychee", "true"}, + {"51536000000", "15", "3147483648", "235.213", "watermelon", "true"}, + }; + + private void checkHeader(ResultSetMetaData resultSetMetaData, String[] title) + throws SQLException { + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(title[i - 1], resultSetMetaData.getColumnName(i)); + } + } + + private void testNormalOrderBy(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualNum = resultSet.getString(2); + String actualBigNum = resultSet.getString(3); + double actualFloatNum = resultSet.getDouble(4); + String actualStr = resultSet.getString(5); + String actualBool = resultSet.getString(6); + + // System.out.println(actualTime + "," + actualNum + "," + actualBigNum + "," + + // actualFloatNum); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void orderByTest1() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 as t where t.device='d1' order by t.num"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest2() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 t where t.device='d1' order by bigNum,t.time"; + int[] ans = {13, 11, 10, 3, 1, 5, 4, 7, 9, 8, 2, 12, 0, 6, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest3() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by floatNum"; + int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest4() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest5() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by num desc"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + ArrayUtils.reverse(ans); + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest6() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bigNum desc, time asc"; + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest7() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by floatNum desc"; + int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; + ArrayUtils.reverse(ans); + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest8() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str desc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + ArrayUtils.reverse(ans); + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest15() { + String sql = "select Time,num+bigNum,floatNum from table0 where device='d1' order by str"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "_col1", "floatNum"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + double actualNum = resultSet.getLong(2); + double actualFloat = resultSet.getDouble(3); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Long.parseLong(RES[ans[i]][2]), actualNum, 0.0001); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloat, 0.0001); + + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + // 2. Multi-level order by test + @Test + public void orderByTest9() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bool asc, str asc"; + int[] ans = {3, 12, 0, 8, 7, 10, 6, 11, 2, 5, 9, 13, 4, 1, 14}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest10() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bool asc, num desc"; + int[] ans = {12, 11, 10, 6, 3, 8, 7, 0, 14, 13, 4, 9, 5, 1, 2}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest11() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by bigNum desc, floatNum desc"; + int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest12() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str desc, floatNum desc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void orderByTest13() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by num+floatNum desc, floatNum desc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + testNormalOrderBy(sql, ans); + } + + @Test + public void orderByTest14() { + String sql = + "select Time,num+bigNum from table0 where device='d1' order by num+floatNum desc, floatNum desc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "_col1"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + double actualNum = resultSet.getLong(2); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Long.parseLong(RES[ans[i]][2]), actualNum, 0.001); + + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void orderByTest16() { + String sql = + "select Time,num+floatNum from table0 where device='d1' order by floatNum+num desc, floatNum desc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "_col1"}); + int i = 0; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + double actualNum = resultSet.getDouble(2); + + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Double.parseDouble(RES[ans[i]][3]), + actualNum, + 0.001); + + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void orderByTest17() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str desc, str asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void orderByTest18() { + String sql = + "select Time,num,bigNum,floatNum,str,bool from table0 where device='d1' order by str, str"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderBy(sql, ans); + } + + // limit cannot be pushed down in ORDER BY + @Test + public void orderByTest19() { + String sql = "select Time,num from table0 where device='d1' order by num limit 5"; + int[] ans = {2, 1, 0, 7, 8}; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader(metaData, new String[] {"Time", "num"}); + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + String actualNum = resultSet.getString(2); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + // 3. aggregation query + @Ignore + @Test + public void orderByInAggregationTest() { + String sql = "select avg(num) from root.sg.d group by session(10000ms) order by avg(num) desc"; + double[][] ans = new double[][] {{15.0}, {13.0}, {13.0}, {11.0}, {6.4}, {4.6}}; + long[] times = + new long[] {51536000000L, 41536000000L, 41536900000L, 31536100000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest2() { + String sql = + "select avg(num) from root.sg.d group by session(10000ms) order by max_value(floatNum)"; + double[][] ans = + new double[][] { + {13.0, 54.12}, + {11.0, 54.121}, + {13.0, 231.34}, + {15.0, 235.213}, + {6.4, 1231.21}, + {4.6, 4654.231} + }; + long[] times = + new long[] {41536900000L, 31536100000L, 41536000000L, 51536000000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest3() { + String sql = + "select avg(num) from root.sg.d group by session(10000ms) order by avg(num) desc,max_value(floatNum)"; + double[] ans = new double[] {15.0, 13.0, 13.0, 11.0, 6.4, 4.6}; + long[] times = + new long[] {51536000000L, 41536900000L, 41536000000L, 31536100000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest4() { + String sql = + "select avg(num)+avg(floatNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; + double[][] ans = + new double[][] {{1079.56122}, {395.4584}, {65.121}, {151.2855}, {67.12}, {250.213}}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest5() { + String sql = + "select min_value(bigNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; + long[] ans = + new long[] {2147483646L, 2147483650L, 2147468648L, 2146483648L, 2107483648L, 3147483648L}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + long actualMinValue = resultSet.getLong(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]], actualMinValue, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest6() { + String sql = + "select min_value(num)+min_value(bigNum) from root.sg.d group by session(10000ms) order by avg(num)+avg(floatNum)"; + long[] ans = + new long[] {2147483647L, 2147483654L, 2147468659L, 2146483660L, 2107483661L, 3147483663L}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualMinValue = resultSet.getDouble(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]], actualMinValue, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest7() { + String sql = + "select avg(num)+min_value(floatNum) from root.sg.d group by session(10000ms) order by max_value(floatNum)"; + double[][] ans = + new double[][] { + {13.0, 54.12, 54.12}, + {11.0, 54.121, 54.121}, + {13.0, 231.34, 45.231}, + {15.0, 235.213, 235.213}, + {6.4, 1231.21, 56.32}, + {4.6, 4654.231, 12.123} + }; + long[] times = + new long[] {41536900000L, 31536100000L, 41536000000L, 51536000000L, 31536000000L, 0L}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[i], actualTime); + assertEquals(ans[i][0] + ans[i][2], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationTest8() { + String sql = + "select avg(num)+avg(floatNum) from root.sg.d group by session(10000ms) order by avg(floatNum)+avg(num)"; + double[][] ans = + new double[][] {{1079.56122}, {395.4584}, {65.121}, {151.2855}, {67.12}, {250.213}}; + long[] times = + new long[] {0L, 31536000000L, 31536100000L, 41536000000L, 41536900000L, 51536000000L}; + int[] order = new int[] {2, 4, 3, 5, 1, 0}; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + double actualAvg = resultSet.getDouble(2); + assertEquals(times[order[i]], actualTime); + assertEquals(ans[order[i]][0], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + // 4. raw data query with align by device + private void testNormalOrderByAlignByDevice(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d1"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + if (device.equals("d1")) { + device = "d2"; + } else { + device = "d1"; + i++; + } + total++; + } + assertEquals(i, ans.length); + assertEquals(total, ans.length * 2); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void alignByDeviceOrderByTest1() { + String sql = + "select Time,device,num+bigNum from table0 order by num+floatNum desc, floatNum desc, device asc"; + int[] ans = {4, 5, 1, 14, 12, 6, 0, 9, 7, 13, 10, 8, 11, 3, 2}; + String device = "d1"; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + double actualNum = resultSet.getLong(3); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals( + Long.parseLong(RES[ans[i]][1]) + Long.parseLong(RES[ans[i]][2]), actualNum, 0.0001); + if (device.equals("d1")) { + device = "d2"; + } else { + device = "d1"; + i++; + } + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void alignByDeviceOrderByTest2() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by num, device asc"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest3() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by floatNum, device asc"; + int[] ans = {2, 3, 11, 13, 10, 8, 7, 9, 0, 12, 6, 14, 1, 5, 4}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest4() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by str, device asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest5() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by num desc, device asc"; + int[] ans = {2, 1, 0, 7, 8, 5, 9, 4, 3, 6, 10, 11, 13, 12, 14}; + testNormalOrderByAlignByDevice(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void alignByDeviceOrderByTest6() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by str desc, device asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderByAlignByDevice(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + @Test + public void alignByDeviceOrderByTest7() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bool asc, num desc, device asc"; + int[] ans = {12, 11, 10, 6, 3, 8, 7, 0, 14, 13, 4, 9, 5, 1, 2}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest8() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, floatNum desc, device asc"; + int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderByAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest9() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by str desc, floatNum desc, device asc"; + int[] ans = {3, 2, 5, 12, 0, 9, 13, 8, 4, 7, 1, 10, 6, 11, 14}; + testNormalOrderByAlignByDevice(sql, org.bouncycastle.util.Arrays.reverse(ans)); + } + + private void testNormalOrderByMixAlignBy(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d1"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + if (device.equals("d2")) { + i++; + device = "d1"; + } else { + device = "d2"; + } + + total++; + } + assertEquals(total, ans.length * 2); + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + private void testDeviceViewOrderByMixAlignBy(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d2"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + i++; + total++; + if (total == ans.length) { + i = 0; + if (device.equals("d2")) { + device = "d1"; + } else { + device = "d2"; + } + } + } + assertEquals(total, ans.length * 2); + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + private void orderByBigNumAlignByDevice(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute(USE_DB); + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + checkHeader( + metaData, new String[] {"Time", "device", "num", "bigNum", "floatNum", "str", "bool"}); + int i = 0; + int total = 0; + String device = "d1"; + while (resultSet.next()) { + + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + String actualNum = resultSet.getString(3); + String actualBigNum = resultSet.getString(4); + double actualFloatNum = resultSet.getDouble(5); + String actualStr = resultSet.getString(6); + String actualBool = resultSet.getString(7); + + if (total < 4) { + i = total % 2; + if (total < 2) { + device = "d2"; + } else { + device = "d1"; + } + } + + assertEquals(device, actualDevice); + assertEquals(Long.parseLong(RES[ans[i]][0]), actualTime); + assertEquals(RES[ans[i]][1], actualNum); + assertEquals(RES[ans[i]][2], actualBigNum); + assertEquals(Double.parseDouble(RES[ans[i]][3]), actualFloatNum, 0.0001); + assertEquals(RES[ans[i]][4], actualStr); + assertEquals(RES[ans[i]][5], actualBool); + + if (device.equals("d2")) { + device = "d1"; + } else { + i++; + device = "d2"; + } + + total++; + } + assertEquals(i, ans.length); + assertEquals(total, ans.length * 2); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void alignByDeviceOrderByTest12() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, device desc, time asc"; + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + orderByBigNumAlignByDevice(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest13() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, time desc, device asc"; + int[] ans = {14, 6, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + testNormalOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest14() { + int[] ans = {14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by time desc, bigNum desc, device asc"; + testNormalOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest15() { + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by device desc, bigNum desc, time asc"; + testDeviceViewOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest16() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by device desc, time asc, bigNum desc"; + int[] ans = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; + testDeviceViewOrderByMixAlignBy(sql, ans); + } + + @Test + public void alignByDeviceOrderByTest17() { + String sql = + "select Time,device,num,bigNum,floatNum,str,bool from table0 order by bigNum desc, device desc, num asc, time asc"; + int[] ans = {6, 14, 0, 12, 2, 8, 9, 7, 4, 5, 1, 3, 10, 11, 13}; + orderByBigNumAlignByDevice(sql, ans); + } + + // 5. aggregation query align by device + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest() { + String sql = + "select avg(num) from root.** group by session(10000ms) order by avg(num) align by device"; + + double[] ans = {4.6, 4.6, 6.4, 6.4, 11.0, 11.0, 13.0, 13.0, 13.0, 13.0, 15.0, 15.0}; + long[] times = + new long[] { + 0L, + 0L, + 31536000000L, + 31536000000L, + 31536100000L, + 31536100000L, + 41536000000L, + 41536900000L, + 41536000000L, + 41536900000L, + 51536000000L, + 51536000000L + }; + String[] device = + new String[] { + "root.sg.d", + "root.sg.d2", + "root.sg.d", + "root.sg.d2", + "root.sg.d", + "root.sg.d2", + "root.sg.d", + "root.sg.d", + "root.sg.d2", + "root.sg.d2", + "root.sg.d", + "root.sg.d2" + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + long actualTime = resultSet.getLong(1); + String actualDevice = resultSet.getString(2); + double actualAvg = resultSet.getDouble(3); + + assertEquals(device[i], actualDevice); + assertEquals(times[i], actualTime); + assertEquals(ans[i], actualAvg, 0.0001); + i++; + } + assertEquals(i, ans.length); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + @Ignore + public void orderByInAggregationAlignByDeviceTest2() { + String sql = "select avg(num) from root.** order by avg(num) align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + private void checkSingleDouble(String sql, Object value, boolean deviceAsc) { + String device = "root.sg.d"; + if (!deviceAsc) device = "root.sg.d2"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + String deviceName = resultSet.getString(1); + double actualVal = resultSet.getDouble(2); + assertEquals(deviceName, device); + assertEquals(Double.parseDouble(value.toString()), actualVal, 1); + if (device.equals("root.sg.d")) device = "root.sg.d2"; + else device = "root.sg.d"; + i++; + } + assertEquals(i, 2); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest3() { + String sql = + "select avg(num)+avg(bigNum) from root.** order by max_value(floatNum) align by device"; + long value = 2388936669L + 8; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest4() { + + String sql = + "select avg(num)+avg(bigNum) from root.** order by max_value(floatNum)+min_value(num) align by device"; + long value = 2388936669L + 8; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest5() { + String sql = + "select avg(num) from root.** order by max_value(floatNum)+avg(num) align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest6() { + String sql = + "select avg(num) from root.** order by max_value(floatNum)+avg(num), device asc, time desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest7() { + String sql = + "select avg(num) from root.** order by max_value(floatNum)+avg(num), time asc, device desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, false); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest8() { + String sql = + "select avg(num) from root.** order by time asc, max_value(floatNum)+avg(num), device desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, false); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest9() { + String sql = + "select avg(num) from root.** order by device asc, max_value(floatNum)+avg(num), time desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest10() { + String sql = + "select avg(num) from root.** order by max_value(floatNum) desc,time asc, avg(num) asc, device desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, false); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest11() { + String sql = + "select avg(num) from root.** order by max_value(floatNum) desc,device asc, avg(num) asc, time desc align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest12() { + String sql = + "select avg(num+floatNum) from root.** order by time,avg(num+floatNum) align by device"; + String value = "537.34154"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest13() { + String sql = "select avg(num) from root.** order by time,avg(num+floatNum) align by device"; + String value = "8"; + checkSingleDouble(sql, value, true); + } + + @Ignore + @Test + public void orderByInAggregationAlignByDeviceTest14() { + String sql = "select avg(num+floatNum) from root.** order by time,avg(num) align by device"; + String value = "537.34154"; + checkSingleDouble(sql, value, true); + } + + String[][] UDFRes = + new String[][] { + {"0", "3", "0", "0"}, + {"20", "2", "0", "0"}, + {"40", "1", "0", "0"}, + {"80", "9", "0", "0"}, + {"100", "8", "0", "0"}, + {"31536000000", "6", "0", "0"}, + {"31536000100", "10", "0", "0"}, + {"31536000500", "4", "0", "0"}, + {"31536001000", "5", "0", "0"}, + {"31536010000", "7", "0", "0"}, + {"31536100000", "11", "0", "0"}, + {"41536000000", "12", "2146483648", "0"}, + {"41536000020", "14", "0", "14"}, + {"41536900000", "13", "2107483648", "0"}, + {"51536000000", "15", "0", "15"}, + }; + + // UDF Test + private void orderByUDFTest(String sql, int[] ans) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + String time = resultSet.getString(1); + String num = resultSet.getString(2); + String topK = resultSet.getString(3); + String bottomK = resultSet.getString(4); + + assertEquals(time, UDFRes[ans[i]][0]); + assertEquals(num, UDFRes[ans[i]][1]); + if (Objects.equals(UDFRes[ans[i]][3], "0")) { + assertNull(topK); + } else { + assertEquals(topK, UDFRes[ans[i]][3]); + } + + if (Objects.equals(UDFRes[ans[i]][2], "0")) { + assertNull(bottomK); + } else { + assertEquals(bottomK, UDFRes[ans[i]][2]); + } + + i++; + } + assertEquals(i, 15); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void orderByUDFTest1() { + String sql = + "select num, top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2') from root.sg.d order by top_k(num, 'k'='2') nulls first, bottom_k(bigNum, 'k'='2') nulls first"; + int[] ans = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 11, 12, 14}; + orderByUDFTest(sql, ans); + } + + @Ignore + @Test + public void orderByUDFTest2() { + String sql = + "select num, top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2') from root.sg.d order by top_k(num, 'k'='2'), bottom_k(bigNum, 'k'='2')"; + int[] ans = {12, 14, 13, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + orderByUDFTest(sql, ans); + } + + private void errorTest(String sql, String error) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.executeQuery(sql); + } catch (Exception e) { + assertEquals(error, e.getMessage()); + } + } + + @Ignore + @Test + public void errorTest1() { + errorTest( + "select num from root.sg.d order by avg(bigNum)", + "701: Raw data and aggregation hybrid query is not supported."); + } + + @Ignore + @Test + public void errorTest2() { + errorTest( + "select avg(num) from root.sg.d order by bigNum", + "701: Raw data and aggregation hybrid query is not supported."); + } + + @Test + public void errorTest3() { + errorTest( + "select bigNum,floatNum from table0 order by s1", + "700: Error occurred while parsing SQL to physical plan: line 1:28 no viable alternative at input 'select bigNum,floatNum from table0'"); + } + + @Ignore + @Test + public void errorTest4() { + errorTest( + "select bigNum,floatNum from root.** order by bigNum", + "701: root.**.bigNum in order by clause shouldn't refer to more than one timeseries."); + } + + @Ignore + @Test + public void errorTest7() { + errorTest( + "select last bigNum,floatNum from root.** order by root.sg.d.bigNum", + "701: root.sg.d.bigNum in order by clause doesn't exist in the result of last query."); + } + + // last query + public void testLastQueryOrderBy(String sql, String[][] ans) { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int i = 0; + while (resultSet.next()) { + String time = resultSet.getString(1); + String num = resultSet.getString(2); + String value = resultSet.getString(3); + String dataType = resultSet.getString(4); + + assertEquals(time, ans[0][i]); + assertEquals(num, ans[1][i]); + assertEquals(value, ans[2][i]); + assertEquals(dataType, ans[3][i]); + + i++; + } + assertEquals(i, 4); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Ignore + @Test + public void lastQueryOrderBy() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d.num", "root.sg.d2.num", "root.sg.d.bigNum", "root.sg.d2.bigNum"}, + {"15", "15", "3147483648", "3147483648"}, + {"INT32", "INT32", "INT64", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by value, timeseries"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy2() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d2.bigNum", "root.sg.d.num", "root.sg.d.bigNum"}, + {"15", "3147483648", "15", "3147483648"}, + {"INT32", "INT64", "INT32", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by timeseries desc"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy3() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d2.bigNum", "root.sg.d.num", "root.sg.d.bigNum"}, + {"15", "3147483648", "15", "3147483648"}, + {"INT32", "INT64", "INT32", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by timeseries desc, value asc"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy4() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d.num", "root.sg.d2.bigNum", "root.sg.d.bigNum"}, + {"15", "15", "3147483648", "3147483648"}, + {"INT32", "INT32", "INT64", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by value, timeseries desc"; + testLastQueryOrderBy(sql, ans); + } + + @Ignore + @Test + public void lastQueryOrderBy5() { + String[][] ans = + new String[][] { + {"51536000000", "51536000000", "51536000000", "51536000000"}, + {"root.sg.d2.num", "root.sg.d.num", "root.sg.d2.bigNum", "root.sg.d.bigNum"}, + {"15", "15", "3147483648", "3147483648"}, + {"INT32", "INT32", "INT64", "INT64"} + }; + String sql = "select last bigNum,num from root.** order by datatype, timeseries desc"; + testLastQueryOrderBy(sql, ans); + } + + private static List TIMES = + Arrays.asList( + 0L, + 20L, + 40L, + 80L, + 100L, + 31536000000L, + 31536000100L, + 31536000500L, + 31536001000L, + 31536010000L, + 31536100000L, + 41536000000L, + 41536000020L, + 41536900000L, + 51536000000L); + + protected static void sessionInsertData1() { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE \"db0\""); + session.executeNonQueryStatement("USE \"db0\""); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("device", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("num", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("bigNum", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("floatNum", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("str", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("bool", TSDataType.BOOLEAN)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + List fieldIds = IMeasurementSchema.getMeasurementNameList(schemaList); + List dataTypes = IMeasurementSchema.getDataTypeList(schemaList); + + List values = + Arrays.asList( + new Object[] {"d1", "a1", 3, 2947483648L, 231.2121, "coconut", false}, + new Object[] {"d1", "a1", 2, 2147483648L, 434.12, "pineapple", true}, + new Object[] {"d1", "a1", 1, 2247483648L, 12.123, "apricot", true}, + new Object[] {"d1", "a1", 9, 2147483646L, 43.12, "apple", false}, + new Object[] {"d1", "a1", 8, 2147483964L, 4654.231, "papaya", true}, + new Object[] {"d1", "a1", 6, 2147483650L, 1231.21, "banana", true}, + new Object[] {"d1", "a1", 10, 3147483648L, 231.55, "pumelo", false}, + new Object[] {"d1", "a1", 4, 2147493648L, 213.1, "peach", false}, + new Object[] {"d1", "a1", 5, 2149783648L, 56.32, "orange", false}, + new Object[] {"d1", "a1", 7, 2147983648L, 213.112, "lemon", true}, + new Object[] {"d1", "a1", 11, 2147468648L, 54.121, "pitaya", false}, + new Object[] {"d1", "a1", 12, 2146483648L, 45.231, "strawberry", false}, + new Object[] {"d1", "a1", 14, 2907483648L, 231.34, "cherry", false}, + new Object[] {"d1", "a1", 13, 2107483648L, 54.12, "lychee", true}, + new Object[] {"d1", "a1", 15, 3147483648L, 235.213, "watermelon", true}); + Tablet tablet = new Tablet("table0", fieldIds, dataTypes, columnTypes, TIMES.size()); + for (int i = 0; i < TIMES.size(); i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, TIMES.get(i)); + for (int j = 0; j < schemaList.size(); j++) { + tablet.addValue(schemaList.get(j).getMeasurementName(), rowIndex, values.get(i)[j]); + } + } + session.insert(tablet); + } catch (Exception e) { + e.printStackTrace(); + } + } + + protected static void sessionInsertData2() { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db0\""); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("device", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("num", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("bigNum", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("floatNum", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("str", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("bool", TSDataType.BOOLEAN)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + List fieldIds = IMeasurementSchema.getMeasurementNameList(schemaList); + List dataTypes = IMeasurementSchema.getDataTypeList(schemaList); + List values = + Arrays.asList( + new Object[] {"d2", "a2", 3, 2947483648L, 231.2121, "coconut", false}, + new Object[] {"d2", "a2", 2, 2147483648L, 434.12, "pineapple", true}, + new Object[] {"d2", "a2", 1, 2247483648L, 12.123, "apricot", true}, + new Object[] {"d2", "a2", 9, 2147483646L, 43.12, "apple", false}, + new Object[] {"d2", "a2", 8, 2147483964L, 4654.231, "papaya", true}, + new Object[] {"d2", "a2", 6, 2147483650L, 1231.21, "banana", true}, + new Object[] {"d2", "a2", 10, 3147483648L, 231.55, "pumelo", false}, + new Object[] {"d2", "a2", 4, 2147493648L, 213.1, "peach", false}, + new Object[] {"d2", "a2", 5, 2149783648L, 56.32, "orange", false}, + new Object[] {"d2", "a2", 7, 2147983648L, 213.112, "lemon", true}, + new Object[] {"d2", "a2", 11, 2147468648L, 54.121, "pitaya", false}, + new Object[] {"d2", "a2", 12, 2146483648L, 45.231, "strawberry", false}, + new Object[] {"d2", "a2", 14, 2907483648L, 231.34, "cherry", false}, + new Object[] {"d2", "a2", 13, 2107483648L, 54.12, "lychee", true}, + new Object[] {"d2", "a2", 15, 3147483648L, 235.213, "watermelon", true}); + Tablet tablet = new Tablet("table0", fieldIds, dataTypes, columnTypes, TIMES.size()); + for (int i = 0; i < TIMES.size(); i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, TIMES.get(i)); + for (int j = 0; j < schemaList.size(); j++) { + tablet.addValue(schemaList.get(j).getMeasurementName(), rowIndex, values.get(i)[j]); + } + } + session.insert(tablet); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/query/IoTDBArithmeticTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/query/IoTDBArithmeticTableViewIT.java new file mode 100644 index 0000000000000..9a6091e516d0c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/query/IoTDBArithmeticTableViewIT.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.query; + +import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBArithmeticTableViewIT { + + private static final double E = 0.0001; + + private static final String DATABASE_NAME = "test"; + + private static final long MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000; + + private static final double[][] BASE_ANS = {{1, 1, 1.0, 1.0}, {2, 2, 2.0, 2.0}, {3, 3, 3.0, 3.0}}; + + private static final Map> OPERATIONS = new HashMap<>(); + + static { + OPERATIONS.put(" + ", (a, b) -> a + b); + OPERATIONS.put(" - ", (a, b) -> a - b); + OPERATIONS.put(" * ", (a, b) -> a * b); + OPERATIONS.put(" / ", (a, b) -> a / b); + OPERATIONS.put(" % ", (a, b) -> a % b); + } + + private static final String[] INSERTION_SQLS = { + "CREATE DATABASE root.test", + "CREATE TIMESERIES root.sg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s2 WITH DATATYPE=INT64, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s3 WITH DATATYPE=FLOAT, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s4 WITH DATATYPE=DOUBLE, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s5 WITH DATATYPE=DATE, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s6 WITH DATATYPE=TIMESTAMP, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s7 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.d1.s8 WITH DATATYPE=TEXT, ENCODING=PLAIN", + "insert into root.sg.d1(time, s1, s2, s3, s4, s5, s6, s7, s8) values (1, 1, 1, 1.0, 1.0, '2024-01-01', 10, true, 'test')", + "insert into root.sg.d1(time, s1, s2, s3, s4, s5, s6, s7, s8) values (2, 2, 2, 2.0, 2.0, '2024-02-01', 20, true, 'test')", + "insert into root.sg.d1(time, s1, s2, s3, s4, s5, s6, s7, s8) values (3, 3, 3, 3.0, 3.0, '2024-03-01', 30, true, 'test')", + "CREATE ALIGNED TIMESERIES root.sg2.d1(date DATE)", + "insert into root.sg2.d1(time, date) values (1, '9999-12-31')", + "insert into root.sg2.d1(time, date) values (2, '1000-01-01')" + }; + + private static final String[] CREATE_TABLE_VIEW_SQLS = { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW table1 (device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 DATE FIELD, s6 TIMESTAMP FIELD, s7 BOOLEAN FIELD, s8 TEXT FIELD) as root.sg.**", + "CREATE VIEW table2 (device STRING TAG, date DATE FIELD) as root.sg2.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(INSERTION_SQLS); + prepareTableData(CREATE_TABLE_VIEW_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + public static double[][] calculateExpectedAns(String operator) { + double[][] expectedAns = new double[BASE_ANS.length][BASE_ANS[0].length * BASE_ANS[0].length]; + BiFunction operation = OPERATIONS.get(operator); + if (operation == null) { + throw new IllegalArgumentException("Unsupported operator: " + operator); + } + + for (int i = 0; i < BASE_ANS.length; i++) { + int baseLength = BASE_ANS[i].length; + for (int j = 0; j < baseLength; j++) { + for (int k = 0; k < baseLength; k++) { + expectedAns[i][j * baseLength + k] = operation.apply(BASE_ANS[i][j], BASE_ANS[i][k]); + } + } + } + return expectedAns; + } + + @Test + public void testArithmeticBinaryWithoutDateAndTimestamp() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + + // generate sql + + for (String operator : new String[] {" + ", " - ", " * ", " / ", " % "}) { + List expressions = new ArrayList<>(); + String[] operands = new String[] {"s1", "s2", "s3", "s4"}; + for (String leftOperand : operands) { + for (String rightOperand : operands) { + expressions.add(leftOperand + operator + rightOperand); + } + } + + String sql = String.format("select %s from table1", String.join(",", expressions)); + ResultSet resultSet = statement.executeQuery(sql); + + // generate answer + double[][] expectedAns = calculateExpectedAns(operator); + + // Make sure the number of columns in the result set is correct + assertEquals(expressions.size(), resultSet.getMetaData().getColumnCount()); + + // check the result + for (double[] expectedAn : expectedAns) { + resultSet.next(); + for (int i = 0; i < expectedAn.length; i++) { + double result = Double.parseDouble(resultSet.getString(i + 1)); + assertEquals(expectedAn[i], result, E); + } + } + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testArithmeticBinaryWithDateAndTimestamp() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + + String sql = + "select s1+s5,s1+s6,s2+s5,s2+s6,s5+s1,s5+s2,s6+s1,s6+s2,s5-s1,s5-s2,s6-s1,s6-s2 from table1"; + ResultSet resultSet = statement.executeQuery(sql); + + assertEquals(12, resultSet.getMetaData().getColumnCount()); + + int[][] expectedAns = { + {20240101, 11, 20240101, 11, 20240101, 20240101, 11, 11, 20231231, 20231231, 9, 9}, + {20240201, 22, 20240201, 22, 20240201, 20240201, 22, 22, 20240131, 20240131, 18, 18}, + {20240301, 33, 20240301, 33, 20240301, 20240301, 33, 33, 20240229, 20240229, 27, 27} + }; + + for (int[] expectedAn : expectedAns) { + resultSet.next(); + + for (int i = 0; i < expectedAn.length; i++) { + int result = resultSet.getInt(i + 1); + assertEquals(expectedAn[i], result); + } + } + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testArithmeticUnary() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE test"); + + String[] expressions = new String[] {"- s1", "- s2", "- s3", "- s4"}; + String sql = String.format("select %s from table1", String.join(",", expressions)); + ResultSet resultSet = statement.executeQuery(sql); + + assertEquals(expressions.length, resultSet.getMetaData().getColumnCount()); + + for (int i = 1; i < 4; ++i) { + resultSet.next(); + for (int j = 0; j < expressions.length; ++j) { + double expected = -i; + double actual = Double.parseDouble(resultSet.getString(j + 1)); + assertEquals(expected, actual, E); + } + } + resultSet.close(); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testTimestampNegation() { + String sql = "select -s6 from table1"; + tableResultSetEqualTest( + sql, + new String[] {"_col0"}, + new String[] { + "1969-12-31T23:59:59.990Z,", "1969-12-31T23:59:59.980Z,", "1969-12-31T23:59:59.970Z," + }, + DATABASE_NAME); + } + + @Test + public void testBinaryWrongType() { + + testBinaryDifferentCombinationsFail( + new String[] {" * ", " / ", " % "}, + new String[] {"s1", "s2", "s3", "s4"}, + new String[] {"s5", "s6"}, + "table1", + "701: Cannot apply operator", + "test"); + + testBinaryDifferentCombinationsFail( + new String[] {" * ", " / ", " % "}, + new String[] {"s5", "s6"}, + new String[] {"s1", "s2", "s3", "s4"}, + "table1", + "701: Cannot apply operator", + "test"); + } + + @Test + public void testUnaryWrongType() { + tableAssertTestFail("select -s5 from table1", "701: Cannot negate", "test"); + tableAssertTestFail("select -s7 from table1", "701: Cannot negate", "test"); + tableAssertTestFail("select -s8 from table1", "701: Cannot negate", "test"); + } + + @Test + public void testOverflow() { + // int addition overflow + tableAssertTestFail( + String.format("select s1+%d from table1", Integer.MAX_VALUE), + "750: int Addition overflow", + "test"); + + // int subtraction overflow + tableAssertTestFail( + String.format("select s1-(%d) from table1", Integer.MIN_VALUE), + "750: int Subtraction overflow", + "test"); + + // int multiplication overflow + tableAssertTestFail( + String.format("select (s1+1)*%d from table1", Integer.MAX_VALUE), + "750: int Multiplication overflow", + "test"); + + // Date addition overflow + tableAssertTestFail( + String.format("select s5+%d from table1", Long.MAX_VALUE), + "750: long Addition overflow", + "test"); + + // Date subtraction overflow + tableAssertTestFail( + String.format("select s5-(%d) from table1", Long.MIN_VALUE), + "750: long Subtraction overflow", + "test"); + + // long addition overflow + tableAssertTestFail( + String.format("select s2+%d from table1", Long.MAX_VALUE), + "750: long Addition overflow", + "test"); + + // long subtraction overflow + tableAssertTestFail( + String.format("select s2-(%d) from table1", Long.MIN_VALUE), + "750: long Subtraction overflow", + "test"); + + // Timestamp addition overflow + tableAssertTestFail( + String.format("select s6+%d from table1", Long.MAX_VALUE), + "750: long Addition overflow", + "test"); + + // Timestamp subtraction overflow + tableAssertTestFail( + String.format("select s6-(%d) from table1", Long.MIN_VALUE), + "750: long Subtraction overflow", + "test"); + } + + @Test + public void testFloatDivisionByZeroSpecialCase() { + String[] expectedHeader = new String[5]; + for (int i = 0; i < expectedHeader.length; i++) { + expectedHeader[i] = "_col" + i; + } + String[] expectedAns = { + "Infinity,0.0,NaN,-0.0,-Infinity,", + "Infinity,0.0,NaN,-0.0,-Infinity,", + "Infinity,0.0,NaN,-0.0,-Infinity," + }; + + tableResultSetEqualTest( + "select s3/0.0,0.0/s3,0.0/0.0,0.0/-s3,-s3/0.0 from table1", + expectedHeader, + expectedAns, + "test"); + } + + @Test + public void testDoubleDivisionByZeroSpecialCase() { + String[] expectedHeader = new String[5]; + for (int i = 0; i < expectedHeader.length; i++) { + expectedHeader[i] = "_col" + i; + } + String[] expectedAns = {"NaN,0.0,NaN,0.0,NaN,", "NaN,0.0,NaN,0.0,NaN,", "NaN,0.0,NaN,0.0,NaN,"}; + + tableResultSetEqualTest( + "select s3%0.0,0.0%s3,0.0%0.0,0.0%-s3,-s3%0.0 from table1", + expectedHeader, expectedAns, "test"); + } + + @Test + public void testDivisionByZero() { + tableAssertTestFail("select s1/0 from table1", "751: Division by zero", "test"); + tableAssertTestFail("select s2/0 from table1", "751: Division by zero", "test"); + + tableAssertTestFail("select s1%0 from table1", "751: Division by zero", "test"); + tableAssertTestFail("select s2%0 from table1", "751: Division by zero", "test"); + } + + @Test + public void testDateOutOfRange() { + tableAssertTestFail( + String.format( + "select date + %s from table2 where time = 1", + DateTimeUtils.correctPrecision(MILLISECONDS_IN_DAY)), + "752: Year must be between 1000 and 9999.", + "test"); + + tableAssertTestFail( + String.format( + "select %s + date from table2 where time = 1", + DateTimeUtils.correctPrecision(MILLISECONDS_IN_DAY)), + "752: Year must be between 1000 and 9999.", + "test"); + + tableAssertTestFail( + String.format("select %s + date from table2 where time = 1", 86400000), + "752: Year must be between 1000 and 9999.", + "test"); + + tableAssertTestFail( + String.format( + "select date - %s from table2 where time = 2", + DateTimeUtils.correctPrecision(MILLISECONDS_IN_DAY)), + "752: Year must be between 1000 and 9999.", + "test"); + } + + private void testBinaryDifferentCombinationsFail( + String[] operators, + String[] leftOperands, + String[] rightOperands, + String tableName, + String errMsg, + String databaseName) { + for (String operator : operators) { + for (String leftOperand : leftOperands) { + for (String rightOperand : rightOperands) { + tableAssertTestFail( + String.format( + "select %s %s %s from %s", leftOperand, operator, rightOperand, tableName), + errMsg, + databaseName); + } + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/query/IoTDBPaginationTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/query/IoTDBPaginationTableViewIT.java new file mode 100644 index 0000000000000..a1f074a261172 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/query/IoTDBPaginationTableViewIT.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBPaginationTableViewIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE root.vehicle", + "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE", + "CREATE TIMESERIES root.vehicle.d0.s1 WITH DATATYPE=INT64, ENCODING=RLE", + "CREATE TIMESERIES root.vehicle.d0.s2 WITH DATATYPE=FLOAT, ENCODING=RLE", + "insert into root.vehicle.d0(timestamp,s0) values(1,101)", + "insert into root.vehicle.d0(timestamp,s0) values(2,198)", + "insert into root.vehicle.d0(timestamp,s0) values(100,99)", + "insert into root.vehicle.d0(timestamp,s0) values(101,99)", + "insert into root.vehicle.d0(timestamp,s0) values(102,80)", + "insert into root.vehicle.d0(timestamp,s0) values(103,99)", + "insert into root.vehicle.d0(timestamp,s0) values(104,90)", + "insert into root.vehicle.d0(timestamp,s0) values(105,99)", + "insert into root.vehicle.d0(timestamp,s0) values(106,99)", + "insert into root.vehicle.d0(timestamp,s0) values(2,10000)", + "insert into root.vehicle.d0(timestamp,s0) values(50,10000)", + "insert into root.vehicle.d0(timestamp,s0) values(1000,22222)", + "insert into root.vehicle.d0(timestamp,s1) values(1,1101)", + "insert into root.vehicle.d0(timestamp,s1) values(2,198)", + "insert into root.vehicle.d0(timestamp,s1) values(100,199)", + "insert into root.vehicle.d0(timestamp,s1) values(101,199)", + "insert into root.vehicle.d0(timestamp,s1) values(102,180)", + "insert into root.vehicle.d0(timestamp,s1) values(103,199)", + "insert into root.vehicle.d0(timestamp,s1) values(104,190)", + "insert into root.vehicle.d0(timestamp,s1) values(105,199)", + "insert into root.vehicle.d0(timestamp,s1) values(2,40000)", + "insert into root.vehicle.d0(timestamp,s1) values(50,50000)", + "insert into root.vehicle.d0(timestamp,s1) values(1000,55555)", + "insert into root.vehicle.d0(timestamp,s2) values(1000,55555)", + "insert into root.vehicle.d0(timestamp,s2) values(2,2.22)", + "insert into root.vehicle.d0(timestamp,s2) values(3,3.33)", + "insert into root.vehicle.d0(timestamp,s2) values(4,4.44)", + "insert into root.vehicle.d0(timestamp,s2) values(102,10.00)", + "insert into root.vehicle.d0(timestamp,s2) values(105,11.11)", + "insert into root.vehicle.d0(timestamp,s2) values(1000,1000.11)", + "insert into root.vehicle.d0(timestamp,s1) values(2000-01-01T08:00:00+08:00, 100)", + "CREATE DATABASE root.db", + "CREATE TIMESERIES root.db.d1.s1 INT32", + "insert into root.db.d1(timestamp,s1) values(0, 0)", + "insert into root.db.d1(timestamp,s1) values(1, 1)", + "insert into root.db.d1(timestamp,s1) values(2, 2)", + "insert into root.db.d1(timestamp,s1) values(3, 3)", + "insert into root.db.d1(timestamp,s1) values(4, 4)", + "flush", + "insert into root.db.d1(timestamp,s1) values(5, 5)", + "insert into root.db.d1(timestamp,s1) values(6, 6)", + "insert into root.db.d1(timestamp,s1) values(7, 7)", + "insert into root.db.d1(timestamp,s1) values(8, 8)", + "insert into root.db.d1(timestamp,s1) values(9, 9)", + "flush", + "insert into root.db.d1(timestamp,s1) values(10, 10)", + "insert into root.db.d1(timestamp,s1) values(11, 11)", + "insert into root.db.d1(timestamp,s1) values(12, 12)", + "insert into root.db.d1(timestamp,s1) values(13, 13)", + "insert into root.db.d1(timestamp,s1) values(14, 14)" + }; + private static final String[] CREATE_TABLE_VIEW_SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW vehicle(device STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD) as root.vehicle.**", + "CREATE VIEW db(device STRING TAG, s1 INT32 FIELD) as root.db.**", + }; + + @BeforeClass + public static void setUp() throws InterruptedException { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(SQLs); + prepareTableData(CREATE_TABLE_VIEW_SQLs); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void rawDataQueryTest() { + List querySQLs = + Arrays.asList( + "SELECT time,s1 FROM vehicle WHERE time<200 limit 3", + "SELECT time,s0 FROM vehicle WHERE s1 > 190 limit 3", + "SELECT time,s1,s2 FROM vehicle WHERE s1 > 190 or s2 < 10.0 offset 2 limit 3", + "SELECT time,s2 FROM vehicle WHERE s1 > 190 or s2 < 10.0 offset 1 limit 3"); + List> expectHeaders = + Arrays.asList( + Arrays.asList("time", "s1"), + Arrays.asList("time", "s0"), + Arrays.asList("time", "s1", "s2"), + Arrays.asList("time", "s2")); + List retArrays = + Arrays.asList( + new String[] { + defaultFormatDataTime(1) + ",1101,", + defaultFormatDataTime(2) + ",40000,", + defaultFormatDataTime(50) + ",50000,", + }, + new String[] { + defaultFormatDataTime(1) + ",101,", + defaultFormatDataTime(2) + ",10000,", + defaultFormatDataTime(50) + ",10000," + }, + new String[] { + defaultFormatDataTime(3) + ",null,3.33,", + defaultFormatDataTime(4) + ",null,4.44,", + defaultFormatDataTime(50) + ",50000,null," + }, + new String[] { + defaultFormatDataTime(2) + ",2.22,", + defaultFormatDataTime(3) + ",3.33,", + defaultFormatDataTime(4) + ",4.44,", + }); + + for (int i = 0; i < querySQLs.size(); i++) { + tableResultSetEqualTest( + querySQLs.get(i), + expectHeaders.get(i).toArray(new String[0]), + retArrays.get(i), + DATABASE_NAME); + } + } + + @Test + public void limitOffsetPushDownTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = + new String[] { + "3,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 1 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 3 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "7,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 5 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "12,", + }; + tableResultSetEqualTest( + "select s1 from db where time > 1 offset 10 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "1,", + }; + tableResultSetEqualTest( + "select s1 from db order by time limit 1 offset 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select s1 from db order by device limit 1 offset 1", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBColumnsMatchTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBColumnsMatchTableViewIT.java new file mode 100644 index 0000000000000..91e3beb3dbfa2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBColumnsMatchTableViewIT.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBColumnsMatchTableViewIT { + private static final String TREE_DB_NAME = "root.test"; + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.test.d1(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.d2(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.d3(s1 INT32, s2 INT64)", + "INSERT INTO root.test.d1(time,s1,s2) aligned values(1,1,10)", + "INSERT INTO root.test.d2(time,s1,s2) aligned values(2,2,20)", + "INSERT INTO root.test.d3(time,s1) aligned values(3,3)", + }; + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD) as root.test.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void columnsTest() { + + // case 1: match all columns + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,10,", + "1970-01-01T00:00:00.002Z,d2,2,20,", + "1970-01-01T00:00:00.003Z,d3,3,null," + }; + tableResultSetEqualTest( + "SELECT COLUMNS(*) FROM table1 order by time", expectedHeader, retArray, DATABASE_NAME); + + // case 2: match columns which name starts with 's' + expectedHeader = new String[] {"s1", "s2"}; + retArray = new String[] {"1,10,", "2,20,", "3,null,"}; + tableResultSetEqualTest( + "SELECT COLUMNS('^s.*') FROM table1 order by s1", expectedHeader, retArray, DATABASE_NAME); + + // case 3: cannot match any column + tableAssertTestFail( + "SELECT COLUMNS('^b.*') FROM table1", + "701: No matching columns found that match regex '^b.*'", + DATABASE_NAME); + + // case 4: invalid input of regex + tableAssertTestFail( + "SELECT COLUMNS('*b') FROM table1 order by s1", "701: Invalid regex '*b'", DATABASE_NAME); + } + + @Test + public void aliasTest() { + // case 1: normal test + String[] expectedHeader = new String[] {"series1", "series2"}; + String[] retArray = new String[] {"1,10,", "2,20,", "3,null,"}; + tableResultSetEqualTest( + "SELECT COLUMNS('^s(.*)') AS \"series$1\" FROM table1 order by series1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"s", "s", "s", "s"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,10,", + "1970-01-01T00:00:00.002Z,d2,2,20,", + "1970-01-01T00:00:00.003Z,d3,3,null," + }; + tableResultSetEqualTest( + "SELECT COLUMNS(*) AS s FROM table1 order by 1", expectedHeader, retArray, DATABASE_NAME); + + // case 2: no according reference group + tableAssertTestFail( + "SELECT COLUMNS('^s.*') AS \"series$1\" FROM table1", "701: No group 1", DATABASE_NAME); + + // case 3: alias grammar error + tableAssertTestFail( + "SELECT COLUMNS('^s.*') AS 'series$1' FROM table1", + "700: line 1:27: mismatched input ''series$1''. Expecting: ", + DATABASE_NAME); + } + + @Test + public void usedWithExpressionTest1() { + // case 1: select min value for each column + String[] expectedHeader = + new String[] {"_col0_time", "_col1_device_id", "_col2_s1", "_col3_s2"}; + String[] retArray = new String[] {"1970-01-01T00:00:00.001Z,d1,1,10,"}; + tableResultSetEqualTest( + "SELECT min(COLUMNS(*)) FROM table1", expectedHeader, retArray, DATABASE_NAME); + + // case 2: multi columns() in different selectItem, + expectedHeader = + new String[] { + "_col0_time", "_col1_device_id", "_col2_s1", "_col3_s2", "_col4_s1", "_col5_s2" + }; + retArray = new String[] {"1970-01-01T00:00:00.001Z,d1,1,10,1,10,"}; + tableResultSetEqualTest( + "SELECT min(COLUMNS(*)), min(COLUMNS('^s.*')) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: multi columns() in one expression, columns() are same + expectedHeader = new String[] {"_col0_s1", "_col1_s2"}; + retArray = new String[] {"4,30,"}; + tableResultSetEqualTest( + "SELECT min(COLUMNS('^s.*')) + max(COLUMNS('^s.*')) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: multi columns() in one expression, columns() are different + tableAssertTestFail( + "SELECT min(COLUMNS('^s.*')) + max(COLUMNS('^t.*')) FROM table1", + "701: Multiple different COLUMNS in the same expression are not supported", + DATABASE_NAME); + + // case 5: get last row of Data + expectedHeader = + new String[] {"_col0", "_col1_time", "_col2_device_id", "_col3_s1", "_col4_s2"}; + retArray = new String[] {"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,d3,3,null,"}; + tableResultSetEqualTest( + "SELECT last(time), last_by(COLUMNS(*), time) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: get last non-null value of each column, and according time + expectedHeader = + new String[] { + "_col0_time", + "_col1_device_id", + "_col2_s1", + "_col3_s2", + "_col4_time", + "_col5_device_id", + "_col6_s1", + "_col7_s2" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d3,3,20,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.002Z," + }; + tableResultSetEqualTest( + "SELECT last(COLUMNS(*)), last_by(time,COLUMNS(*)) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "test", "test", "test", "test", "_col4_time", "_col5_device_id", "_col6_s1", "_col7_s2" + }; + tableResultSetEqualTest( + "SELECT last(COLUMNS(*)) as test, last_by(time,COLUMNS(*)) FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + // used with all supported Expressions + @Test + public void usedWithExpressionTest2() { + String[] expectedHeader = + new String[] { + "_col0_s1", + "_col1_s2", + "_col2_s1", + "_col3_s2", + "_col4_s1", + "_col5_s2", + "_col6_s1", + "_col7_s2" + }; + String[] retArray = + new String[] { + "2,20,-1,-10,true,false,false,true,", + "4,40,-2,-20,true,false,true,true,", + "6,null,-3,null,false,null,true,false," + }; + tableResultSetEqualTest( + "select columns('s.*')+columns('s.*'), -columns('s.*'),columns('s.*') between 1 and 2,if(columns('s.*')>1,true,false) from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "select columns(*).s1 from table1", + "701: Columns are not supported in DereferenceExpression", + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + retArray = new String[] {"1970-01-01T00:00:00.002Z,d2,2,20,"}; + tableResultSetEqualTest( + "select * from table1 where columns('s.*') > any(select s1 from table1) order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0_s1", + "_col1_s2", + "_col2_s1", + "_col3_time", + "_col4_device_id", + "_col5_s1", + "_col6_s2", + "_col7_time", + "_col8_device_id", + "_col9_s1", + "_col10_s2", + "_col11_time", + "_col12_device_id", + "_col13_s1", + "_col14_s2" + }; + retArray = + new String[] { + "1,10,1,true,true,true,true,true,true,true,true,false,false,false,false,", + "2,20,2,true,true,true,true,true,true,true,true,false,false,false,false,", + "3,null,3,true,true,true,null,true,true,true,false,false,false,false,true," + }; + tableResultSetEqualTest( + "select trim(cast(columns('s.*') as String)),COALESCE(columns('s1.*'),1), columns(*) in (columns(*)),columns(*) is not null, columns(*) is null from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0_device_id", "_col1_time", "_col2_device_id", "_col3_s1", "_col4_s2"}; + retArray = + new String[] { + "true,true,true,true,true,", "true,true,true,true,true,", "true,true,true,true,null," + }; + tableResultSetEqualTest( + "select columns('d.*') like 'd_',columns(*)=columns(*) from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "SELECT CASE columns('s.*') WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'many' END from table1", + "701: CASE operand type does not match WHEN clause operand type: INT64 vs INT32", + DATABASE_NAME); + + expectedHeader = new String[] {"_col0_s1"}; + retArray = new String[] {"one,", "two,", "many,"}; + tableResultSetEqualTest( + "SELECT CASE columns('s1.*') WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'many' END from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0_s1", "_col1_s2"}; + retArray = new String[] {"one,many,", "two,many,", "many,many,"}; + tableResultSetEqualTest( + "select case when columns('s.*') = 1 then 'one' when columns('s.*')=2 then 'two' else 'many' end from table1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void usedInWhereTest() { + // case 1: normal test + String[] expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + String[] retArray = new String[] {"1970-01-01T00:00:00.002Z,d2,2,20,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE COLUMNS('^s.*') > 1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s1", "s2"}; + retArray = + new String[] {"1970-01-01T00:00:00.002Z,d2,2,20,", "1970-01-01T00:00:00.003Z,d3,3,null,"}; + tableResultSetEqualTest( + "SELECT * FROM table1 WHERE COLUMNS('^s1.*') > 1 order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: cannot match any column + tableAssertTestFail( + "SELECT * FROM table1 WHERE COLUMNS('^b.*') > 1", + "701: No matching columns found that match regex '^b.*'", + DATABASE_NAME); + + // case 4: invalid input of regex + tableAssertTestFail( + "SELECT * FROM table1 WHERE COLUMNS('*b') > 1", "701: Invalid regex '*b'", DATABASE_NAME); + } + + @Test + public void otherExceptionTest() { + // cannot be used in HAVING clause + tableAssertTestFail( + "SELECT device_id, count(s1) FROM table1 HAVING COLUMNS('device_id') > 1", + "701: Columns only support to be used in SELECT and WHERE clause", + DATABASE_NAME); + + // cannot be used for columns without name + tableAssertTestFail( + "SELECT COLUMNS(*) FROM (SELECT s1,s1+1 from table1)", + "701: Unknown ColumnName", + DATABASE_NAME); + tableAssertTestFail( + "SELECT COLUMNS('s1') FROM (SELECT s1,s1+1 from table1)", + "701: Unknown ColumnName", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBComplexQueryTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBComplexQueryTableViewIT.java new file mode 100644 index 0000000000000..96f0a9de59632 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBComplexQueryTableViewIT.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBComplexQueryTableViewIT { + protected static final String TREE_DB_NAME = "root.test"; + protected static final String DATABASE_NAME = "test_db"; + + protected static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "create aligned timeseries root.test.employees.D001(name TEXT,Gender TEXT,Status BOOLEAN,employee_id INT32,salary DOUBLE,date_of_birth DATE,Contac_info string)", + "create aligned timeseries root.test.employees.D001(name TEXT,Gender TEXT,Status BOOLEAN,employee_id INT32,salary DOUBLE,date_of_birth DATE,Contac_info string)", + "create aligned timeseries root.test.employees.D002(name TEXT,Gender TEXT,Status BOOLEAN,employee_id INT32,salary DOUBLE,date_of_birth DATE,Contac_info string)", + "create aligned timeseries root.test.employees.D002(name TEXT,Gender TEXT,Status BOOLEAN,employee_id INT32,salary DOUBLE,date_of_birth DATE,Contac_info string)", + "create aligned timeseries root.test.employees.D003(name TEXT,Gender TEXT,Status BOOLEAN,employee_id INT32,salary DOUBLE,date_of_birth DATE,Contac_info string)", + "create aligned timeseries root.test.departments.D001(department_id STRING,dep_name TEXT,dep_phone TEXT,dep_status BOOLEAN,dep_member INT32,employee_id INT32)", + "create aligned timeseries root.test.departments.D002(department_id STRING,dep_name TEXT,dep_phone TEXT,dep_status BOOLEAN,dep_member INT32,employee_id INT32)", + "create aligned timeseries root.test.departments.D003(department_id STRING,dep_name TEXT,dep_phone TEXT,dep_status BOOLEAN,dep_member INT32,employee_id INT32)", + "create aligned timeseries root.test.departments.D004(department_id STRING,dep_name TEXT,dep_phone TEXT,dep_status BOOLEAN,dep_member INT32,employee_id INT32)", + "insert into root.test.employees.D001(time, name, gender, status, employee_id, salary, date_of_birth, contac_info) aligned values(1, 'Mary','Female', false, 1223, 5500.22, '1988-10-12', '133-1212-1234')", + "insert into root.test.employees.D001(time, name, gender, status, employee_id, salary, date_of_birth, contac_info) aligned values(2, 'John', 'Male', true, 40012, 8822, '1985-06-15', '130-1002-1334')", + "insert into root.test.employees.D002(time, name, gender, status, employee_id, salary, date_of_birth, contac_info) aligned values(3, 'Nancy', 'Female', true, 30112, 10002, '1983-08-15', '135-1302-1354')", + "insert into root.test.employees.D002(time, name, gender, status, employee_id, salary, date_of_birth, contac_info) aligned values(4, 'Jack', 'Male', false, 12212, 7000, '1990-03-26', '138-1012-1353')", + "insert into root.test.employees.D003(time, name, gender, status, employee_id, salary, date_of_birth, contac_info) aligned values(5, 'Linda', 'Female', false, 10212, 5600, '1995-06-15', '150-2003-1355')", + "insert into root.test.departments.D001(time, dep_name, dep_phone, dep_status, dep_member,employee_id) aligned values(1,'销售部', '010-2271-2120', false, 1223,1223)", + "insert into root.test.departments.D001(time, dep_name, dep_phone, dep_status, dep_member,employee_id) aligned values(2,'销售部', '010-2271-2120', false, 102, 40012)", + "insert into root.test.departments.D002(time, dep_name, dep_phone, dep_status, dep_member,employee_id) aligned values(3,'客服部', '010-2077-2520', true, 220, 30112)", + "insert into root.test.departments.D002(time, dep_name, dep_phone, dep_status, dep_member,employee_id) aligned values(4,'客服部', '010-2077-2520', true, 2012, 12212)", + "insert into root.test.departments.D003(time, dep_name, dep_phone, dep_status, dep_member,employee_id) aligned values(5,'研发部', '010-3272-2310', true, 300, 10212)", + "insert into root.test.departments.D004(time, dep_name, dep_phone, dep_status, dep_member,employee_id) aligned values(6,'人事部', '010-3272-2312', true, 300, 10200)", + "FLUSH", + }; + protected static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create view employees(department_id STRING TAG,name TEXT FIELD,Gender TEXT FIELD,Status BOOLEAN FIELD,employee_id INT32 FIELD,salary DOUBLE FIELD,date_of_birth DATE FIELD,Contac_info string FIELD) as root.test.employees.**", + "create view departments(department_id STRING TAG,dep_name TEXT FIELD,dep_phone TEXT FIELD,dep_status BOOLEAN FIELD,dep_member INT32 FIELD,employee_id INT32 FIELD) as root.test.departments.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void queryTest1() { + // Look for the non-intersecting departments in the two tables + String[] expectedHeader = new String[] {"department_id", "dep_name"}; + String[] retArray = new String[] {"D004,人事部,"}; + tableResultSetEqualTest( + "select department_id, dep_name from departments where not exists(" + + "select 1 from employees where employees.department_id = departments.department_id)", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBDistinctTagTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBDistinctTagTableViewIT.java new file mode 100644 index 0000000000000..c5fc888927588 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBDistinctTagTableViewIT.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDistinctTagTableViewIT { + private static final String DATABASE_NAME = "test"; + + protected static final String[] SQLs = + new String[] { + "CREATE DATABASE root.test", + // test flush + "CREATE TIMESERIES root.test.t1.d1.s1 INT64", + "CREATE TIMESERIES root.test.t1.d2.s1 INT64", + "CREATE TIMESERIES root.test.t1.d3.s1 INT64", + "CREATE TIMESERIES root.test.t1.d4.s1 INT64", + "insert into root.test.t1.d1(time, s1) values(1000, 10)", + "insert into root.test.t1.d2(time, s1) values(1000, 11)", + "insert into root.test.t1.d2(time, s1) values(2000, 12)", + "insert into root.test.t1.d1(time, s1) values(4000, 13)", + "flush", + "insert into root.test.t1.d3(time, s1) values(5000, 10)", + "insert into root.test.t1.d4(time, s1) values(6000, 11)", + "insert into root.test.t1.d2(time, s1) values(3000, 12)", + "insert into root.test.t1.d1(time, s1) values(2000, 13)", + "flush", + + // test memory + "CREATE TIMESERIES root.test.t2.d1.s1 INT64", + "CREATE TIMESERIES root.test.t2.d2.s1 INT64", + "CREATE TIMESERIES root.test.t2.d3.s1 INT64", + "CREATE TIMESERIES root.test.t2.d4.s1 INT64", + "insert into root.test.t2.d1(time, s1) values(1000, 10)", + "insert into root.test.t2.d2(time, s1) values(1000, 11)", + "insert into root.test.t2.d2(time, s1) values(2000, 12)", + "insert into root.test.t2.d1(time, s1) values(4000, 13)", + "insert into root.test.t2.d3(time, s1) values(5000, 10)", + "insert into root.test.t2.d4(time, s1) values(6000, 11)", + "insert into root.test.t2.d2(time, s1) values(3000, 12)", + "insert into root.test.t2.d1(time, s1) values(2000, 13)", + }; + + protected static final String[] CREATE_TABLE_VIEW_SQLs = + new String[] { + "CREATE DATABASE IF NOT EXISTS test", + "USE test", + "CREATE VIEW t1(deviceId STRING TAG, s1 INT64 FIELD) as root.test.t1.**", + "CREATE VIEW t2(deviceId STRING TAG, s1 INT64 FIELD) as root.test.t2.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDistinct() { + // distinct(deviceId) + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {"d1,", "d2,", "d3,", "d4,"}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t1 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(deviceId) from test.t2 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDistinctWithTimeFilter() { + // distinct(deviceId) ... where time > 3000; + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {"d1,", "d3,", "d4,"}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t1 where time > 3000 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(deviceId) from test.t2 where time > 3000 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDistinctWithPushDownFilter() { + // distinct(deviceId) ... where s1 = 11; + String[] expectedHeader = new String[] {"deviceId"}; + String[] retArray = new String[] {"d2,", "d4,"}; + tableResultSetEqualTest( + "select distinct(deviceId) from test.t1 where s1 = 11 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select distinct(deviceId) from test.t2 where s1 = 11 order by deviceId", + expectedHeader, + retArray, + DATABASE_NAME); + } + + private static void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + fail(e.getMessage()); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : CREATE_TABLE_VIEW_SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBFillTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBFillTableViewIT.java new file mode 100644 index 0000000000000..68dfaf2717600 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBFillTableViewIT.java @@ -0,0 +1,740 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFillTableViewIT { + private static final String TREE_DB_NAME = "root.test"; + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.test.table1.d1(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "INSERT INTO root.test.table1.d1(time,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " aligned values(1, 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO root.test.table1.d1(time,s1,s2,s3,s4,s5) " + + " aligned values(2, 2, 22, 2.2, 22.2, false)", + "INSERT INTO root.test.table1.d1(time,s6,s7,s8,s9,s10) " + + " aligned values(3, 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO root.test.table1.d1(time,s6,s7,s8,s9,s10) " + + " aligned values(4, 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO root.test.table1.d1(time,s1,s2,s3,s4,s5) " + + " aligned values(5, 5, 55, 5.5, 55.5, false)", + "flush", + "INSERT INTO root.test.table1.d1(time,s1,s2,s3,s4,s5) " + + " aligned values(7, 7, 77, 7.7, 77.7, true)", + "INSERT INTO root.test.table1.d1(time,s6,s7,s8,s9,s10) " + + " aligned values(8, 'text8', 'string8', X'cafebabe08', 8, '2024-10-08')", + "CREATE ALIGNED TIMESERIES root.test.table2.shanghai.d1(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.table2.shanghai.d2(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.table2.beijing.d1(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.table2.beijing.d2(s1 INT32, s2 INT64)", + "INSERT INTO root.test.table2.shanghai.d1(time,s2) aligned values(1, 02111)", + "INSERT INTO root.test.table2.shanghai.d1(time,s1) aligned values(2, 0212)", + "INSERT INTO root.test.table2.beijing.d1(time,s2) aligned values(1, 01011)", + "INSERT INTO root.test.table2.beijing.d1(time,s1) aligned values(2, 0102)", + "INSERT INTO root.test.table2.beijing.d1(time,s1) aligned values(3, 0103)", + "INSERT INTO root.test.table2.beijing.d1(time,s1) aligned values(4, 0104)", + "INSERT INTO root.test.table2.beijing.d2(time,s1) aligned values(1, 0101)", + "INSERT INTO root.test.table2.beijing.d2(time,s2) aligned values(2, 01022)", + "INSERT INTO root.test.table2.beijing.d2(time,s2) aligned values(3, 01033)", + "INSERT INTO root.test.table2.beijing.d2(time,s2) aligned values(4, 01044)", + "CREATE ALIGNED TIMESERIES root.test.table3.shanghai.d1(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.table3.beijing.d1(s1 INT32, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.test.table3.beijing.d2(s1 INT32, s2 INT64)", + "INSERT INTO root.test.table3.shanghai.d1(time,s2) aligned values(1, 02111)", + "INSERT INTO root.test.table3.shanghai.d1(time,s1) aligned values(2, 0212)", + "INSERT INTO root.test.table3.beijing.d1(time,s2) aligned values(1, 01011)", + "INSERT INTO root.test.table3.beijing.d1(time,s1) aligned values(2, 0102)", + "INSERT INTO root.test.table3.beijing.d1(time,s1,s2) aligned values(3, 0103,01033)", + "INSERT INTO root.test.table3.beijing.d1(time,s1) aligned values(4, 0104)", + "INSERT INTO root.test.table3.beijing.d2(time,s1) aligned values(1, 0101)", + "INSERT INTO root.test.table3.beijing.d2(time,s2) aligned values(2, 01022)", + "INSERT INTO root.test.table3.beijing.d2(time,s1,s2) aligned values(3, 0103, 01033)", + "INSERT INTO root.test.table3.beijing.d2(time,s2) aligned values(4, 01044)", + }; + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD) as root.test.table1.**", + "CREATE VIEW table2(city STRING TAG, device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD) as root.test.table2.**", + "CREATE VIEW table3(city STRING TAG, device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD) as root.test.table3.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void normalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS", expectedHeader, retArray, DATABASE_NAME); + + // case 2: all with time filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: all without time filter using previous fill with timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: all without time filter using previous fill with timeDuration with helper column + // index + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: all without time filter using previous fill with timeDuration with helper column + // index + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,1,11,1.1,11.1,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 11", + expectedHeader, + retArray, + DATABASE_NAME); + + // case7: all without time filter using previous fill with order by time desc + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1 order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using previous fill with order by value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1 order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using previous fill with subQuery + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.003Z,d1,5,55,5.5,55.5,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select * from table1 order by time desc) FILL METHOD PREVIOUS TIME_BOUND 2ms order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // case10: all without time filter using previous fill with FILL_GROUP + expectedHeader = new String[] {"time", "city", "device_id", "s1", "s2"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,beijing,d1,null,1011,", + "1970-01-01T00:00:00.002Z,beijing,d1,102,1011,", + "1970-01-01T00:00:00.003Z,beijing,d1,103,1011,", + "1970-01-01T00:00:00.004Z,beijing,d1,104,1011,", + "1970-01-01T00:00:00.001Z,beijing,d2,101,null,", + "1970-01-01T00:00:00.002Z,beijing,d2,101,1022,", + "1970-01-01T00:00:00.003Z,beijing,d2,101,1033,", + "1970-01-01T00:00:00.004Z,beijing,d2,101,1044,", + "1970-01-01T00:00:00.001Z,shanghai,d1,null,2111,", + "1970-01-01T00:00:00.002Z,shanghai,d1,212,2111,", + }; + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case11: all without time filter using previous fill with FILL_GROUP and TIME_COLUMN + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS TIME_COLUMN 1 FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case12: all without time filter using previous fill with TIME_BOUND and FILL_GROUP + expectedHeader = new String[] {"time", "city", "device_id", "s1", "s2"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,beijing,d1,null,1011,", + "1970-01-01T00:00:00.002Z,beijing,d1,102,1011,", + "1970-01-01T00:00:00.003Z,beijing,d1,103,1011,", + "1970-01-01T00:00:00.004Z,beijing,d1,104,null,", + "1970-01-01T00:00:00.001Z,beijing,d2,101,null,", + "1970-01-01T00:00:00.002Z,beijing,d2,101,1022,", + "1970-01-01T00:00:00.003Z,beijing,d2,101,1033,", + "1970-01-01T00:00:00.004Z,beijing,d2,null,1044,", + "1970-01-01T00:00:00.001Z,shanghai,d1,null,2111,", + "1970-01-01T00:00:00.002Z,shanghai,d1,212,2111,", + }; + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS TIME_BOUND 2ms FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case13: all without time filter using previous fill with TIME_BOUND, FILL_GROUP and + // TIME_COLUMN + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table2 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1 FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + // case 1: all without time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: all with time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 FILL METHOD LINEAR", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL METHOD LINEAR", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: all without time filter using linear fill with helper column + // index + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR TIME_COLUMN 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: all without time filter using linear fill with helper column index + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR TIME_COLUMN 10", + expectedHeader, + retArray, + DATABASE_NAME); + + // case7: all without time filter using linear fill with order by time desc + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using linear fill with order by value + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL METHOD LINEAR TIME_COLUMN 1 order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using linear fill with subQuery + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 order by time desc) FILL METHOD LINEAR order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // case10: all without time filter using linear fill with FILL_GROUP + expectedHeader = new String[] {"time", "city", "device_id", "s1", "s2"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,beijing,d1,null,1011,", + "1970-01-01T00:00:00.002Z,beijing,d1,102,1022,", + "1970-01-01T00:00:00.003Z,beijing,d1,103,1033,", + "1970-01-01T00:00:00.004Z,beijing,d1,104,null,", + "1970-01-01T00:00:00.001Z,beijing,d2,101,null,", + "1970-01-01T00:00:00.002Z,beijing,d2,102,1022,", + "1970-01-01T00:00:00.003Z,beijing,d2,103,1033,", + "1970-01-01T00:00:00.004Z,beijing,d2,null,1044,", + "1970-01-01T00:00:00.001Z,shanghai,d1,null,2111,", + "1970-01-01T00:00:00.002Z,shanghai,d1,212,null,", + }; + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table3 FILL METHOD LINEAR FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // case11: all without time filter using linear fill with FILL_GROUP and TIME_COLUMN + tableResultSetEqualTest( + "select time,city,device_id,s1,s2 from table3 FILL METHOD LINEAR TIME_COLUMN 1 FILL_GROUP 2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- VALUE FILL --------------------------------- + // case 1: fill with integer value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.003Z,d1,100,100,100.0,100.0,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,100,100,100.0,100.0,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.008Z,d1,100,100,100.0,100.0,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT 100", expectedHeader, retArray, DATABASE_NAME); + + // case 2: fill with float value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.003Z,d1,110,110,110.2,110.2,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,110,110,110.2,110.2,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.008Z,d1,110,110,110.2,110.2,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT 110.2", expectedHeader, retArray, DATABASE_NAME); + + // case 3: fill with boolean value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,false,false,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,0,0,0.0,0.0,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,0,0,0.0,0.0,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,false,false,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,false,false,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,0,0,0.0,0.0,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT false", expectedHeader, retArray, DATABASE_NAME); + + // case 4: fill with string value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT 'iotdb'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT '2018-05-06'", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: fill with blob value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL METHOD CONSTANT X'cafebabe99'", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void abNormalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + tableAssertTestFail( + "select s1 from table1 FILL METHOD PREVIOUS TIME_COLUMN 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Don't need to specify TIME_COLUMN while either TIME_BOUND or FILL_GROUP parameter is not specified", + DATABASE_NAME); + + tableAssertTestFail( + "select s1 from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer TIME_COLUMN for PREVIOUS FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1 from table1 FILL METHOD PREVIOUS FILL_GROUP 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer TIME_COLUMN for PREVIOUS FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of TIME_COLUMN for PREVIOUS FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL TIME_COLUMN position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS TIME_BOUND 2ms TIME_COLUMN 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL TIME_COLUMN position 3 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS FILL_GROUP 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL FILL_GROUP position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD PREVIOUS FILL_GROUP 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL FILL_GROUP position 3 is not in select list", + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + tableAssertTestFail( + "select s1 from table1 FILL METHOD LINEAR", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer TIME_COLUMN for LINEAR FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR TIME_COLUMN 1", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of TIME_COLUMN for LINEAR FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR TIME_COLUMN 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL TIME_COLUMN position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR TIME_COLUMN 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL TIME_COLUMN position 3 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR FILL_GROUP 0", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL FILL_GROUP position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL METHOD LINEAR FILL_GROUP 3", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL FILL_GROUP position 3 is not in select list", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBGapFillTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBGapFillTableViewIT.java new file mode 100644 index 0000000000000..af2edb5e489f8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBGapFillTableViewIT.java @@ -0,0 +1,559 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBGapFillTableViewIT { + private static final String TREE_DB_NAME = "root.table1"; + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.table1.shanghai.d1(s1 DOUBLE, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.table1.beijing.d1(s1 DOUBLE, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.table1.beijing.d2(s1 DOUBLE, s2 INT64)", + "CREATE ALIGNED TIMESERIES root.table1.shanghai.d3(s1 DOUBLE, s2 INT64)", + "INSERT INTO root.table1.shanghai.d1(time,s2) aligned values(2024-09-24T06:15:46.565+00:00, 2)", + "INSERT INTO root.table1.shanghai.d1(time,s1) aligned values(2024-09-24T07:16:15.297+00:00, 27.2)", + "INSERT INTO root.table1.shanghai.d1(time,s1) aligned values(2024-09-24T08:16:21.907+00:00, 27.3)", + "INSERT INTO root.table1.shanghai.d1(time,s1) aligned values(2024-09-24T11:16:28.821+00:00, 29.3)", + "INSERT INTO root.table1.beijing.d2(time,s1) aligned values(2024-09-24T04:14:40.545+00:00, 25.1)", + "INSERT INTO root.table1.beijing.d2(time,s1) aligned values(2024-09-24T11:17:15.297+00:00, 28.2)", + "INSERT INTO root.table1.shanghai.d3(time,s1) aligned values(2024-09-24T08:15:21.907+00:00, 22.3)", + "INSERT INTO root.table1.shanghai.d3(time,s1) aligned values(2024-09-24T08:15:28.821+00:00, 29.3)", + }; + + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create view table1(city STRING TAG, device_id STRING TAG, s1 DOUBLE FIELD, s2 INT64 FIELD) as root.table1.**", + }; + + private static final String MULTI_GAFILL_ERROR_MSG = + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": multiple date_bin_gapfill calls not allowed"; + private static final String TIME_RANGE_CANNOT_INFER_ERROR_MSG = + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": could not infer startTime or endTime from WHERE clause"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void normalGapFillTest() { + + // case 1: avg_s1 of one device without having + String[] expectedHeader = new String[] {"hour_time", "avg_s1"}; + String[] retArray = + new String[] { + "2024-09-24T04:00:00.000Z,null,", + "2024-09-24T05:00:00.000Z,null,", + "2024-09-24T06:00:00.000Z,null,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,27.3,", + "2024-09-24T09:00:00.000Z,null,", + "2024-09-24T10:00:00.000Z,null,", + "2024-09-24T11:00:00.000Z,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,city,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: avg_s1 of one device with having + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 HAVING avg(s1) IS NOT NULL", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,city,device_id HAVING avg(s1) IS NOT NULL", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: avg_s1 of each device + expectedHeader = new String[] {"hour_time", "city", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T05:00:00.000Z,beijing,d2,null,", + "2024-09-24T06:00:00.000Z,beijing,d2,null,", + "2024-09-24T07:00:00.000Z,beijing,d2,null,", + "2024-09-24T08:00:00.000Z,beijing,d2,null,", + "2024-09-24T09:00:00.000Z,beijing,d2,null,", + "2024-09-24T10:00:00.000Z,beijing,d2,null,", + "2024-09-24T11:00:00.000Z,beijing,d2,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,d1,null,", + "2024-09-24T05:00:00.000Z,shanghai,d1,null,", + "2024-09-24T06:00:00.000Z,shanghai,d1,null,", + "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T09:00:00.000Z,shanghai,d1,null,", + "2024-09-24T10:00:00.000Z,shanghai,d1,null,", + "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,", + "2024-09-24T04:00:00.000Z,shanghai,d3,null,", + "2024-09-24T05:00:00.000Z,shanghai,d3,null,", + "2024-09-24T06:00:00.000Z,shanghai,d3,null,", + "2024-09-24T07:00:00.000Z,shanghai,d3,null,", + "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T09:00:00.000Z,shanghai,d3,null,", + "2024-09-24T10:00:00.000Z,shanghai,d3,null,", + "2024-09-24T11:00:00.000Z,shanghai,d3,null,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,device_id order by city,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time),city,device_id order by city,device_id,date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 order by 2,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: avg_s1 of all devices + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,25.1,", + "2024-09-24T05:00:00.000Z,null,", + "2024-09-24T06:00:00.000Z,null,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,26.3,", + "2024-09-24T09:00:00.000Z,null,", + "2024-09-24T10:00:00.000Z,null,", + "2024-09-24T11:00:00.000Z,28.75,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: avg_s1 of all cities + expectedHeader = new String[] {"hour_time", "city", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,25.1,", + "2024-09-24T05:00:00.000Z,beijing,null,", + "2024-09-24T06:00:00.000Z,beijing,null,", + "2024-09-24T07:00:00.000Z,beijing,null,", + "2024-09-24T08:00:00.000Z,beijing,null,", + "2024-09-24T09:00:00.000Z,beijing,null,", + "2024-09-24T10:00:00.000Z,beijing,null,", + "2024-09-24T11:00:00.000Z,beijing,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,null,", + "2024-09-24T05:00:00.000Z,shanghai,null,", + "2024-09-24T06:00:00.000Z,shanghai,null,", + "2024-09-24T07:00:00.000Z,shanghai,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,26.3,", + "2024-09-24T09:00:00.000Z,shanghai,null,", + "2024-09-24T10:00:00.000Z,shanghai,null,", + "2024-09-24T11:00:00.000Z,shanghai,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: avg_s1 of all device_ids + expectedHeader = new String[] {"hour_time", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,d1,null,", + "2024-09-24T05:00:00.000Z,d1,null,", + "2024-09-24T06:00:00.000Z,d1,null,", + "2024-09-24T07:00:00.000Z,d1,27.2,", + "2024-09-24T08:00:00.000Z,d1,27.3,", + "2024-09-24T09:00:00.000Z,d1,null,", + "2024-09-24T10:00:00.000Z,d1,null,", + "2024-09-24T11:00:00.000Z,d1,29.3,", + "2024-09-24T04:00:00.000Z,d2,25.1,", + "2024-09-24T05:00:00.000Z,d2,null,", + "2024-09-24T06:00:00.000Z,d2,null,", + "2024-09-24T07:00:00.000Z,d2,null,", + "2024-09-24T08:00:00.000Z,d2,null,", + "2024-09-24T09:00:00.000Z,d2,null,", + "2024-09-24T10:00:00.000Z,d2,null,", + "2024-09-24T11:00:00.000Z,d2,28.2,", + "2024-09-24T04:00:00.000Z,d3,null,", + "2024-09-24T05:00:00.000Z,d3,null,", + "2024-09-24T06:00:00.000Z,d3,null,", + "2024-09-24T07:00:00.000Z,d3,null,", + "2024-09-24T08:00:00.000Z,d3,25.8,", + "2024-09-24T09:00:00.000Z,d3,null,", + "2024-09-24T10:00:00.000Z,d3,null,", + "2024-09-24T11:00:00.000Z,d3,null,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 7: no data after where + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = new String[] {}; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T00:00:00.000+00:00 AND time < 2024-09-24T06:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void gapFillWithFillClauseTest() { + + // case 1: avg_s1 of one device without having + String[] expectedHeader = new String[] {"hour_time", "avg_s1"}; + String[] retArray = + new String[] { + "2024-09-24T04:00:00.000Z,null,", + "2024-09-24T05:00:00.000Z,null,", + "2024-09-24T06:00:00.000Z,null,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,27.3,", + "2024-09-24T09:00:00.000Z,27.3,", + "2024-09-24T10:00:00.000Z,27.3,", + "2024-09-24T11:00:00.000Z,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,city,device_id FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: avg_s1 of one device with having + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 HAVING avg(s1) IS NOT NULL FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1,device_id,city HAVING avg(s1) IS NOT NULL FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: avg_s1 of each device + expectedHeader = new String[] {"hour_time", "city", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T05:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T06:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T07:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T08:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T09:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T10:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T11:00:00.000Z,beijing,d2,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,d1,null,", + "2024-09-24T05:00:00.000Z,shanghai,d1,null,", + "2024-09-24T06:00:00.000Z,shanghai,d1,null,", + "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T09:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T10:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,", + "2024-09-24T04:00:00.000Z,shanghai,d3,null,", + "2024-09-24T05:00:00.000Z,shanghai,d3,null,", + "2024-09-24T06:00:00.000Z,shanghai,d3,null,", + "2024-09-24T07:00:00.000Z,shanghai,d3,null,", + "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T09:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T10:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T11:00:00.000Z,shanghai,d3,25.8,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,device_id FILL METHOD PREVIOUS FILL_GROUP 2,3 order by city,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time),city,device_id FILL METHOD PREVIOUS FILL_GROUP 2,3 order by city,device_id,date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,shanghai,d1,null,", + "2024-09-24T05:00:00.000Z,shanghai,d1,null,", + "2024-09-24T06:00:00.000Z,shanghai,d1,null,", + "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T09:00:00.000Z,shanghai,d1,27.3,", + "2024-09-24T10:00:00.000Z,shanghai,d1,null,", + "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,", + "2024-09-24T04:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T05:00:00.000Z,beijing,d2,25.1,", + "2024-09-24T06:00:00.000Z,beijing,d2,null,", + "2024-09-24T07:00:00.000Z,beijing,d2,null,", + "2024-09-24T08:00:00.000Z,beijing,d2,null,", + "2024-09-24T09:00:00.000Z,beijing,d2,null,", + "2024-09-24T10:00:00.000Z,beijing,d2,null,", + "2024-09-24T11:00:00.000Z,beijing,d2,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,d3,null,", + "2024-09-24T05:00:00.000Z,shanghai,d3,null,", + "2024-09-24T06:00:00.000Z,shanghai,d3,null,", + "2024-09-24T07:00:00.000Z,shanghai,d3,null,", + "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T09:00:00.000Z,shanghai,d3,25.8,", + "2024-09-24T10:00:00.000Z,shanghai,d3,null,", + "2024-09-24T11:00:00.000Z,shanghai,d3,null,", + }; + // with time bound + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 FILL METHOD PREVIOUS TIME_BOUND 1h FILL_GROUP 2,3 order by 3,2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: avg_s1 of all devices + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,25.1,", + "2024-09-24T05:00:00.000Z,25.1,", + "2024-09-24T06:00:00.000Z,25.1,", + "2024-09-24T07:00:00.000Z,27.2,", + "2024-09-24T08:00:00.000Z,26.3,", + "2024-09-24T09:00:00.000Z,26.3,", + "2024-09-24T10:00:00.000Z,26.3,", + "2024-09-24T11:00:00.000Z,28.75,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 FILL METHOD PREVIOUS order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) FILL METHOD PREVIOUS order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1 FILL METHOD PREVIOUS order by date_bin_gapfill(1h, time)", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY date_bin_gapfill(1h, time) FILL METHOD PREVIOUS order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: avg_s1 of all cities + expectedHeader = new String[] {"hour_time", "city", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,beijing,25.1,", + "2024-09-24T05:00:00.000Z,beijing,25.1,", + "2024-09-24T06:00:00.000Z,beijing,25.1,", + "2024-09-24T07:00:00.000Z,beijing,25.1,", + "2024-09-24T08:00:00.000Z,beijing,25.1,", + "2024-09-24T09:00:00.000Z,beijing,25.1,", + "2024-09-24T10:00:00.000Z,beijing,25.1,", + "2024-09-24T11:00:00.000Z,beijing,28.2,", + "2024-09-24T04:00:00.000Z,shanghai,null,", + "2024-09-24T05:00:00.000Z,shanghai,null,", + "2024-09-24T06:00:00.000Z,shanghai,null,", + "2024-09-24T07:00:00.000Z,shanghai,27.2,", + "2024-09-24T08:00:00.000Z,shanghai,26.3,", + "2024-09-24T09:00:00.000Z,shanghai,26.3,", + "2024-09-24T10:00:00.000Z,shanghai,26.3,", + "2024-09-24T11:00:00.000Z,shanghai,29.3,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 FILL METHOD PREVIOUS TIME_COLUMN 1 FILL_GROUP 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, city, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 FILL METHOD PREVIOUS FILL_GROUP 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: avg_s1 of all device_ids + expectedHeader = new String[] {"hour_time", "device_id", "avg_s1"}; + retArray = + new String[] { + "2024-09-24T04:00:00.000Z,d1,null,", + "2024-09-24T05:00:00.000Z,d1,null,", + "2024-09-24T06:00:00.000Z,d1,null,", + "2024-09-24T07:00:00.000Z,d1,27.2,", + "2024-09-24T08:00:00.000Z,d1,27.3,", + "2024-09-24T09:00:00.000Z,d1,27.3,", + "2024-09-24T10:00:00.000Z,d1,27.3,", + "2024-09-24T11:00:00.000Z,d1,29.3,", + "2024-09-24T04:00:00.000Z,d2,25.1,", + "2024-09-24T05:00:00.000Z,d2,25.1,", + "2024-09-24T06:00:00.000Z,d2,25.1,", + "2024-09-24T07:00:00.000Z,d2,25.1,", + "2024-09-24T08:00:00.000Z,d2,25.1,", + "2024-09-24T09:00:00.000Z,d2,25.1,", + "2024-09-24T10:00:00.000Z,d2,25.1,", + "2024-09-24T11:00:00.000Z,d2,28.2,", + "2024-09-24T04:00:00.000Z,d3,null,", + "2024-09-24T05:00:00.000Z,d3,null,", + "2024-09-24T06:00:00.000Z,d3,null,", + "2024-09-24T07:00:00.000Z,d3,null,", + "2024-09-24T08:00:00.000Z,d3,25.8,", + "2024-09-24T09:00:00.000Z,d3,25.8,", + "2024-09-24T10:00:00.000Z,d3,25.8,", + "2024-09-24T11:00:00.000Z,d3,25.8,", + }; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2 FILL METHOD PREVIOUS FILL_GROUP 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 7: no data after where + expectedHeader = new String[] {"hour_time", "avg_s1"}; + retArray = new String[] {}; + tableResultSetEqualTest( + "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T00:00:00.000+00:00 AND time < 2024-09-24T06:00:00.00+00:00) AND device_id = 'd1' GROUP BY 1 FILL METHOD PREVIOUS", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void abNormalGapFillTest() { + + // case 1: multiple date_bin_gapfill in group by clause + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where device_id = 'd1' group by date_bin_gapfill(1s, time), date_bin_gapfill(2s, time), 2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + MULTI_GAFILL_ERROR_MSG, + DATABASE_NAME); + + // case 2: no time filter + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 3: with or + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.000+00:00) OR device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 4: time filter is not between + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where (time not between 2024-09-24T04:00:00.000+00:00 AND 2024-09-24T12:00:00.000+00:00) AND (device_id NOT IN ('d3')) group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 5: only > + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time > 2024-09-24T04:00:00.000+00:00 AND device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 6: only >= + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time >= 2024-09-24T04:00:00.000+00:00 group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 7: only < + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time < 2024-09-24T12:00:00.000+00:00 AND device_id = 'd1' group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + + // case 8: only <= + tableAssertTestFail( + "select date_bin_gapfill(1s, time), city, device_id, avg(s1)+avg(s2) from table1 where time <= 2024-09-24T12:00:00.000+00:00 group by 1,2,3 having avg(s2) is not null FILL METHOD PREVIOUS FILL_GROUP 2,3 order by 2,3,1", + TIME_RANGE_CANNOT_INFER_ERROR_MSG, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBNullIdQueryTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBNullIdQueryTableViewIT.java new file mode 100644 index 0000000000000..48b9f89e0c6b6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBNullIdQueryTableViewIT.java @@ -0,0 +1,543 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; + +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetFuzzyTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullIdQueryTableViewIT { + private static final String TREE_DB_NAME = "root.test"; + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.test.testNullId(s1 INT32, s2 BOOLEAN, s3 DOUBLE)", + "INSERT INTO root.test.testNullId(time,s1,s2,s3) " + " aligned values(1, 0, false, 11.1)", + "CREATE ALIGNED TIMESERIES root.test.table1.d1(s1 INT32, s2 BOOLEAN, s3 INT64)", + // in seq disk + "INSERT INTO root.test.table1.d1(time,s1,s2,s3) " + "values(1, 1, false, 11)", + "INSERT INTO root.test.table1.d1(time,s1) " + " aligned values(5, 5)", + "FLUSH", + // in uneq disk + "INSERT INTO root.test.table1.d1(time,s1,s2,s3) " + "aligned values(4, 4, true, 44)", + "INSERT INTO root.test.table1.d1(time,s1) " + " aligned values(3, 3)", + "FLUSH", + // in seq memtable + "INSERT INTO root.test.table1.d1(time,s1,s2,s3) " + "aligned values(7, 7, false, 77)", + "INSERT INTO root.test.table1.d1(time,s1) " + "aligned values(6, 6)", + // in unseq memtable + "INSERT INTO root.test.table1.d1(time,s1) " + " aligned values(2, 2)", + }; + + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW testNullId(id1 STRING TAG, id2 STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD, s3 DOUBLE FIELD) as root.test.testNullId.**", + "CREATE VIEW table1(device_id STRING TAG, s1 INT32 FIELD, s2 BOOLEAN FIELD, s3 INT64 FIELD) as root.test.table1.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullFilterTest() throws Exception { + String result = defaultFormatDataTime(1) + ",0,false,11.1"; + try (final Connection connectionIsNull = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connectionIsNull.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + + ResultSet resultSet = statement.executeQuery("select * from testNullId where id1 is null"); + assertTrue(resultSet.next()); + String ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + resultSet = statement.executeQuery("select * from testNullId where id1 is not null"); + assertFalse(resultSet.next()); + + resultSet = statement.executeQuery("select * from testNullId where id1 like '%'"); + assertFalse(resultSet.next()); + + resultSet = + statement.executeQuery("select * from testNullId where id1 is null and id2 is null"); + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // The second time we read from cache + resultSet = + statement.executeQuery("select * from testNullId where id1 is null and id2 is null"); + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test deduplication + resultSet = + statement.executeQuery("select * from testNullId where id1 is null or id2 is null"); + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test constant select item + resultSet = statement.executeQuery("select *, 1 from testNullId"); + result = defaultFormatDataTime(1) + ",null,null,0,false,11.1,1"; + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("id1") + + "," + + resultSet.getString("id2") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3") + + "," + + resultSet.getString("_col6"); + + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test boolean between + resultSet = + statement.executeQuery("select * from testNullId where s2 between false and true"); + result = defaultFormatDataTime(1) + ",null,null,0,false,11.1"; + assertTrue(resultSet.next()); + ans = + resultSet.getString("time") + + "," + + resultSet.getString("id1") + + "," + + resultSet.getString("id2") + + "," + + resultSet.getString("s1") + + "," + + resultSet.getString("s2") + + "," + + resultSet.getString("s3"); + + assertEquals(result, ans); + assertFalse(resultSet.next()); + + // Test boolean not between + resultSet = + statement.executeQuery("select * from testNullId where s2 not between false and true"); + assertFalse(resultSet.next()); + + // Test same column name + resultSet = statement.executeQuery("select time, s1 as a, s2 as a from testNullId"); + result = defaultFormatDataTime(1) + ",0,false"; + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + assertEquals(3, resultSetMetaData.getColumnCount()); + assertEquals("time", resultSetMetaData.getColumnName(1)); + assertEquals(Types.TIMESTAMP, resultSetMetaData.getColumnType(1)); + assertEquals("a", resultSetMetaData.getColumnName(2)); + assertEquals(Types.INTEGER, resultSetMetaData.getColumnType(2)); + assertEquals("a", resultSetMetaData.getColumnName(3)); + assertEquals(Types.BOOLEAN, resultSetMetaData.getColumnType(3)); + + assertTrue(resultSet.next()); + ans = resultSet.getString(1) + "," + resultSet.getString(2) + "," + resultSet.getString(3); + + assertEquals(result, ans); + assertFalse(resultSet.next()); + } + } + + @Test + public void nullSelectTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where time > 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where time < 7", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where time > 1 and time < 7 and s1 >= 3 and s1 <= 5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where s2 is NULL", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 where s2 IS NOT NULL OR s3 IS NOT NULL", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void noMeasurementColumnsSelectTest() { + String[] expectedHeader = new String[] {"time"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,", + "1970-01-01T00:00:00.002Z,", + "1970-01-01T00:00:00.003Z,", + "1970-01-01T00:00:00.004Z,", + "1970-01-01T00:00:00.005Z,", + "1970-01-01T00:00:00.006Z,", + "1970-01-01T00:00:00.007Z," + }; + tableResultSetEqualTest("select time from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"device_id"}; + retArray = new String[] {"d1,", "d1,", "d1,", "d1,", "d1,", "d1,"}; + tableResultSetEqualTest( + "select device_id from table1 where time > 1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,", + "1970-01-01T00:00:00.002Z,d1,", + "1970-01-01T00:00:00.003Z,d1,", + "1970-01-01T00:00:00.004Z,d1,", + "1970-01-01T00:00:00.005Z,d1,", + "1970-01-01T00:00:00.006Z,d1,", + }; + tableResultSetEqualTest( + "select time, device_id from table1 where time < 7", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,", + "1970-01-01T00:00:00.002Z,d1,", + "1970-01-01T00:00:00.003Z,d1,", + "1970-01-01T00:00:00.004Z,d1,", + "1970-01-01T00:00:00.005Z,d1,", + "1970-01-01T00:00:00.006Z,d1,", + }; + tableResultSetEqualTest( + "select time, device_id from table1 where time < 7", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id"}; + retArray = + new String[] { + "d1,", + }; + tableResultSetEqualTest( + "select device_id from table1 where device_id='d1' limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,", "1970-01-01T00:00:00.004Z,", "1970-01-01T00:00:00.005Z,", + }; + tableResultSetEqualTest( + "select time from table1 where time >= 3 and time <= 5 and device_id='d1'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,", "1970-01-01T00:00:00.004Z,", "1970-01-01T00:00:00.005Z,", + }; + tableResultSetEqualTest( + "select time from table1 where time >= 3 and time < 6", + expectedHeader, + retArray, + DATABASE_NAME); + } + + public void limitOffsetTest() { + String[] expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 OFFSET 2 limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 limit 1 OFFSET 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.003Z,d1,null,null,", + "1970-01-01T00:00:00.004Z,d1,true,44,", + "1970-01-01T00:00:00.005Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.007Z,d1,false,77," + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 OFFSET 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "device_id", "s2", "s3"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,false,11,", "1970-01-01T00:00:00.002Z,d1,null,null,", + }; + tableResultSetEqualTest( + "select time, device_id, s2, s3 from table1 limit 2", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void showStatementTest() { + String[] expectedHeader = new String[] {"CurrentSqlDialect"}; + String[] retArray = + new String[] { + "TABLE,", + }; + tableResultSetEqualTest("show current_sql_dialect", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"CurrentUser"}; + retArray = + new String[] { + "root,", + }; + tableResultSetEqualTest("show current_user", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"CurrentDatabase"}; + retArray = + new String[] { + DATABASE_NAME + ",", + }; + tableResultSetEqualTest("show current_database", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"Version", "BuildInfo"}; + tableResultSetFuzzyTest("show version", expectedHeader, 1, DATABASE_NAME); + + expectedHeader = new String[] {"Variable", "Value"}; + tableResultSetFuzzyTest("show variables", expectedHeader, 15, DATABASE_NAME); + + expectedHeader = new String[] {"ClusterId"}; + tableResultSetFuzzyTest("show clusterid", expectedHeader, 1, DATABASE_NAME); + + expectedHeader = new String[] {"ClusterId"}; + tableResultSetFuzzyTest("show cluster_id", expectedHeader, 1, DATABASE_NAME); + + expectedHeader = new String[] {"CurrentTimestamp"}; + tableResultSetFuzzyTest("show current_timestamp", expectedHeader, 1, DATABASE_NAME); + } + + @Test + public void setSqlDialectTest() throws SQLException { + createUser("tempuser", "temppw"); + + try (Connection userCon = EnvFactory.getEnv().getConnection("tempuser", "temppw"); + Statement userStmt = userCon.createStatement()) { + assertCurrentSqlDialect(true, userStmt); + + // set Tree to Table + userStmt.execute("set sql_dialect=table"); + assertCurrentSqlDialect(false, userStmt); + + // set Table to Tree + userStmt.execute("set sql_dialect=tree"); + assertCurrentSqlDialect(true, userStmt); + } + } + + @Test + public void setSqlDialectContextCleanTest() throws SQLException { + try (Connection userCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement userStmt = userCon.createStatement()) { + userStmt.execute("create database test1"); + userStmt.execute("use test1"); + userStmt.execute("set sql_dialect=tree"); + assertCurrentSqlDialect(true, userStmt); + userStmt.execute("insert into root.db(time,s1) values (0,1), (1, 3), (2,5)"); + } + } + + public static void assertCurrentSqlDialect(boolean expectedTree, Statement statement) + throws SQLException { + ResultSet resultSet = statement.executeQuery("show current_sql_dialect"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + assertEquals("CurrentSqlDialect", resultSetMetaData.getColumnName(1)); + int count = 0; + while (resultSet.next()) { + assertEquals(expectedTree ? "TREE" : "TABLE", resultSet.getString(1)); + count++; + } + assertEquals(1, count); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBNullValueTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBNullValueTableViewIT.java new file mode 100644 index 0000000000000..073db94a9b8ec --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBNullValueTableViewIT.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullValueTableViewIT { + private static final String TREE_DB_NAME = "root.test"; + private static final String DATABASE_NAME = "test"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "create timeseries root.test.table1.d1.s1 string", + "insert into root.test.table1.d1(time,s1) values(0, null), (1, 1)", + "flush", + "insert into root.test.table1.d1(time,s1) values(0, 0)", + "flush" + }; + + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create view table1(id1 tag, s1 string) as root.test.table1.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullTest() { + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = new String[] {"time", "id1", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,", + }; + tableResultSetEqualTest("select * from table1", expectedHeader, retArray, DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationFunctionDistinctIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationFunctionDistinctIT.java new file mode 100644 index 0000000000000..2285bc9cf26b8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationFunctionDistinctIT.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +/** + * In this Class, we construct the scenario using DistinctAccumulator for + * AggregationFunctionDistinct. Other cases are covered by {@link IoTDBTableViewAggregationIT}. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableViewAggregationFunctionDistinctIT { + + private static final String TREE_DB_NAME = "root.test"; + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.test.table1.d01(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "INSERT INTO root.test.table1.d01(time,s1,s3,s6,s8,s9) aligned values (2024-09-24T06:15:30.000+00:00,30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.d01(time,s2,s3,s4,s6,s7,s9,s10) aligned values (2024-09-24T06:15:35.000+00:00,35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d01(time,s1,s3,s5,s7,s9) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d01(time,s2,s5,s9,s10) aligned values (2024-09-24T06:15:50.000+00:00,50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d01(time,s1,s2,s3,s4,s6,s7,s9,s10) aligned values (2024-09-24T06:15:35.000+00:00,30,35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "FLUSH", + }; + private static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW table1(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD) as root.test.table1.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void countDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = new String[] {"d01,2,2,3,1,2,2,2,1,4,1,"}; + + tableResultSetEqualTest( + "select device_id, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void countIfDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = new String[] {"d01,0,1,1,1,1,1,1,1,1,1,"}; + tableResultSetEqualTest( + "select device_id, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void avgDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,35.0,42500.0,35.0,35.0,", + }; + tableResultSetEqualTest( + "select device_id, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sumDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = new String[] {"d01,70.0,85000.0,105.0,35.0,"}; + tableResultSetEqualTest( + "select device_id, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + String[] retArray = + new String[] { + "d01,d01,30,35000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,", + }; + + tableResultSetEqualTest( + "select device_id, min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + String[] retArray = + new String[] { + "d01,d01,40,50000,40.0,35.0,true,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe30,2024-09-24T06:15:50.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:35.000Z,", + }; + + tableResultSetEqualTest( + "select device_id, max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z," + }; + tableResultSetEqualTest( + "select device_id, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,40,50000,40.0,35.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe30,2024-09-24T06:15:50.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select device_id, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByDistinctTest() { + String[] expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,2024-09-24T06:15:30.000Z,2024-09-24T06:15:50.000Z,2024-09-24T06:15:35.000Z," + }; + tableResultSetEqualTest( + "select device_id, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extremeDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = new String[] {"d01,40,50000,40.0,35.0,"}; + tableResultSetEqualTest( + "select device_id, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void varianceDistinctTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = new String[] {"d01,25.0,5.625E7,16.7,0.0,"}; + tableResultSetEqualTest( + "select device_id, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java new file mode 100644 index 0000000000000..e3c01f5e6d4ce --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java @@ -0,0 +1,5054 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.buildHeaders; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.repeatTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableViewAggregationIT { + protected static final String TREE_DB_NAME = "root.test"; + protected static final String DATABASE_NAME = "test"; + protected static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.huangpu.d01(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.huangpu.d02(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.huangpu.d03(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.huangpu.d04(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.pudong.d05(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.pudong.d06(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.pudong.d07(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.shanghai.shanghai.pudong.d08(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.chaoyang.d09(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.chaoyang.d10(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.chaoyang.d11(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.chaoyang.d12(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.haidian.d13(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.haidian.d14(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.haidian.d15(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.beijing.beijing.haidian.d16(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d01(time,s1,s3,s6,s8,s9) aligned values (2024-09-24T06:15:30.000+00:00,30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d01(time,s2,s3,s4,s6,s7,s9,s10) aligned values (2024-09-24T06:15:35.000+00:00,35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d01(time,s1,s3,s5,s7,s9) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d01(time,s2,s5,s9,s10) aligned values (2024-09-24T06:15:50.000+00:00,50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d01(time,s1,s4,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d02(time,s1,s5,s6,s7,s9) aligned values (2024-09-24T06:15:36.000+00:00,36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d02(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d02(time,s2,s7,s8,s9) aligned values (2024-09-24T06:15:50.000+00:00,50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d03(time,s2,s8,s9) aligned values (2024-09-24T06:15:31.000+00:00,31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d03(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:36.000+00:00,36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d03(time,s1,s3,s5,s6,s8,s9) aligned values (2024-09-24T06:15:41.000+00:00,41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d03(time,s2,s4,s7,s9) aligned values (2024-09-24T06:15:46.000+00:00,46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d03(time,s3,s6,s9) aligned values (2024-09-24T06:15:51.000+00:00,51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d04(time,s3,s5,s7,s9,s10) aligned values (2024-09-24T06:15:30.000+00:00,30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d04(time,s2,s9) aligned values (2024-09-24T06:15:40.000+00:00,40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.huangpu.d04(time,s1,s4,s6,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d05(time,s1,s3,s6,s8,s9) aligned values (2024-09-24T06:15:30.000+00:00,30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d05(time,s2,s3,s4,s6,s7,s9,s10) aligned values (2024-09-24T06:15:35.000+00:00,35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d05(time,s1,s3,s5,s7,s9) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d05(time,s2,s5,s9,s10) aligned values (2024-09-24T06:15:50.000+00:00,50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d05(time,s1,s4,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d06(time,s1,s5,s6,s7,s9) aligned values (2024-09-24T06:15:36.000+00:00,36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d06(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d06(time,s2,s7,s8,s9) aligned values (2024-09-24T06:15:50.000+00:00,50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d07(time,s2,s8,s9) aligned values (2024-09-24T06:15:31.000+00:00,31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d07(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:36.000+00:00,36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d07(time,s1,s3,s5,s6,s8,s9) aligned values (2024-09-24T06:15:41.000+00:00,41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d07(time,s2,s4,s7,s9) aligned values (2024-09-24T06:15:46.000+00:00,46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d07(time,s3,s6,s9) aligned values (2024-09-24T06:15:51.000+00:00,51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d08(time,s3,s5,s7,s9,s10) aligned values (2024-09-24T06:15:30.000+00:00,30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d08(time,s2,s9) aligned values (2024-09-24T06:15:40.000+00:00,40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.shanghai.shanghai.pudong.d08(time,s1,s4,s6,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d09(time,s1,s3,s6,s8,s9) aligned values (2024-09-24T06:15:30.000+00:00,30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d09(time,s2,s3,s4,s6,s7,s9,s10) aligned values (2024-09-24T06:15:35.000+00:00,35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d09(time,s1,s3,s5,s7,s9) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d09(time,s2,s5,s9,s10) aligned values (2024-09-24T06:15:50.000+00:00,50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d09(time,s1,s4,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d10(time,s1,s5,s6,s7,s9) aligned values (2024-09-24T06:15:36.000+00:00,36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d10(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d10(time,s2,s7,s8,s9) aligned values (2024-09-24T06:15:50.000+00:00,50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d11(time,s2,s8,s9) aligned values (2024-09-24T06:15:31.000+00:00,31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d11(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:36.000+00:00,36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d11(time,s1,s3,s5,s6,s8,s9) aligned values (2024-09-24T06:15:41.000+00:00,41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d11(time,s2,s4,s7,s9) aligned values (2024-09-24T06:15:46.000+00:00,46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d11(time,s3,s6,s9) aligned values (2024-09-24T06:15:51.000+00:00,51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d12(time,s3,s5,s7,s9,s10) aligned values (2024-09-24T06:15:30.000+00:00,30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d12(time,s2,s9) aligned values (2024-09-24T06:15:40.000+00:00,40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.chaoyang.d12(time,s1,s4,s6,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d13(time,s1,s3,s6,s8,s9) aligned values (2024-09-24T06:15:30.000+00:00,30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d13(time,s2,s3,s4,s6,s7,s9,s10) aligned values (2024-09-24T06:15:35.000+00:00,35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d13(time,s1,s3,s5,s7,s9) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d13(time,s2,s5,s9,s10) aligned values (2024-09-24T06:15:50.000+00:00,50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d13(time,s1,s4,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d14(time,s1,s5,s6,s7,s9) aligned values (2024-09-24T06:15:36.000+00:00,36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d14(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:40.000+00:00,40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d14(time,s2,s7,s8,s9) aligned values (2024-09-24T06:15:50.000+00:00,50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d15(time,s2,s8,s9) aligned values (2024-09-24T06:15:31.000+00:00,31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d15(time,s1,s4,s7,s9,s10) aligned values (2024-09-24T06:15:36.000+00:00,36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d15(time,s1,s3,s5,s6,s8,s9) aligned values (2024-09-24T06:15:41.000+00:00,41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d15(time,s2,s4,s7,s9) aligned values (2024-09-24T06:15:46.000+00:00,46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d15(time,s3,s6,s9) aligned values (2024-09-24T06:15:51.000+00:00,51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d16(time,s3,s5,s7,s9,s10) aligned values (2024-09-24T06:15:30.000+00:00,30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d16(time,s2,s9) aligned values (2024-09-24T06:15:40.000+00:00,40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.beijing.beijing.haidian.d16(time,s1,s4,s6,s8,s9) aligned values (2024-09-24T06:15:55.000+00:00,55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "FLUSH", + }; + + protected static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW table1(province STRING TAG, city STRING TAG, region STRING TAG, device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD) as root.test.table1.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ================================================================== + // ==================== Normal Aggregation Test ===================== + // ================================================================== + @Test + public void countTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select count(*) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count('a') from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count(*) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count(1) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(*) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,0,0,1,0,1,0,1,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(s1), count(s2), count(s3), count(s4), count(s5), count(s6), count(s7), count(s8), count(s9), count(s10) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,chaoyang,d10,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,chaoyang,d11,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,chaoyang,d12,1,1,1,1,1,1,1,1,3,1,", + "beijing,beijing,haidian,d13,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,haidian,d14,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,haidian,d15,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,haidian,d16,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,huangpu,d01,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,huangpu,d02,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,huangpu,d03,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,huangpu,d04,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,pudong,d05,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,pudong,d06,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,pudong,d07,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,pudong,d08,1,1,1,1,1,1,1,1,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, count(s1), count(s2), count(s3), count(s4), count(s5), count(s6), count(s7), count(s8), count(s9), count(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,5,", + "beijing,beijing,chaoyang,d10,3,", + "beijing,beijing,chaoyang,d11,5,", + "beijing,beijing,chaoyang,d12,3,", + "beijing,beijing,haidian,d13,5,", + "beijing,beijing,haidian,d14,3,", + "beijing,beijing,haidian,d15,5,", + "beijing,beijing,haidian,d16,3,", + "shanghai,shanghai,huangpu,d01,5,", + "shanghai,shanghai,huangpu,d02,3,", + "shanghai,shanghai,huangpu,d03,5,", + "shanghai,shanghai,huangpu,d04,3,", + "shanghai,shanghai,pudong,d05,5,", + "shanghai,shanghai,pudong,d06,3,", + "shanghai,shanghai,pudong,d07,5,", + "shanghai,shanghai,pudong,d08,3,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,count(*) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,device_id,count(1) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,16,", + "beijing,beijing,haidian,16,", + "shanghai,shanghai,huangpu,16,", + "shanghai,shanghai,pudong,16,", + }; + tableResultSetEqualTest( + "select province,city,region,count(*) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,count(1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,32,", "shanghai,shanghai,32,", + }; + tableResultSetEqualTest( + "select province,city,count(*) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,count(1) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,32,", "shanghai,32,", + }; + tableResultSetEqualTest( + "select province,count(*) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,count(1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest("select count(*) from table1", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest("select count(1) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void countIfTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select count_if(device_id = 'd01') from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest( + "select count_if(true) from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d02,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d02,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d02,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d03,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d03,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d03,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d03,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d03,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d04,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d04,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d04,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d05,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d05,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d05,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d05,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d05,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d06,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d06,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d06,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d07,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d07,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d07,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d07,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d07,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d08,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d08,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d08,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d09,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d09,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d09,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d09,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d09,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d10,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d10,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d10,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d11,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d11,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d11,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d11,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d11,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d12,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d12,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d12,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d13,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d13,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d13,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d13,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d13,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d14,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d14,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d14,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d15,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d15,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d15,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d15,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d15,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d16,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d16,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,0,0,0,0,0,1,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,1,1,1,0,1,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,0,0,0,0,1,1,1,1,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,0,0,0,0,0,1,0,0,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count_if(s1 is null), count_if(s2 < 50000), count_if(s3 > 30), count_if(s4 < 55), count_if(s5), count_if(s6 like '%pudong%'), count_if(s7 = 'shanghai_pudong_red_B_d06_36'), count_if(s8 is null), count(s9 is null), count_if(s10 is not null) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2,1,2,1,1,0,0,3,5,2,", + "beijing,beijing,chaoyang,d10,1,0,0,1,1,0,0,2,3,1,", + "beijing,beijing,chaoyang,d11,3,2,2,2,0,0,0,3,5,1,", + "beijing,beijing,chaoyang,d12,2,1,0,0,1,0,0,2,3,1,", + "beijing,beijing,haidian,d13,2,1,2,1,1,0,0,3,5,2,", + "beijing,beijing,haidian,d14,1,0,0,1,1,0,0,2,3,1,", + "beijing,beijing,haidian,d15,3,2,2,2,0,0,0,3,5,1,", + "beijing,beijing,haidian,d16,2,1,0,0,1,0,0,2,3,1,", + "shanghai,shanghai,huangpu,d01,2,1,2,1,1,0,0,3,5,2,", + "shanghai,shanghai,huangpu,d02,1,0,0,1,1,0,0,2,3,1,", + "shanghai,shanghai,huangpu,d03,3,2,2,2,0,0,0,3,5,1,", + "shanghai,shanghai,huangpu,d04,2,1,0,0,1,0,0,2,3,1,", + "shanghai,shanghai,pudong,d05,2,1,2,1,1,2,0,3,5,2,", + "shanghai,shanghai,pudong,d06,1,0,0,1,1,1,1,2,3,1,", + "shanghai,shanghai,pudong,d07,3,2,2,2,0,2,0,3,5,1,", + "shanghai,shanghai,pudong,d08,2,1,0,0,1,1,0,2,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, count_if(s1 is null), count_if(s2 < 50000), count_if(s3 > 30), count_if(s4 < 55), count_if(s5), count_if(s6 like '%pudong%'), count_if(s7 = 'shanghai_pudong_red_B_d06_36'), count_if(s8 is null), count(s9 is null), count_if(s10 is not null) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,", + "beijing,beijing,haidian,5,", + "shanghai,shanghai,huangpu,5,", + "shanghai,shanghai,pudong,5,", + }; + tableResultSetEqualTest( + "select province,city,region,count(s3 > 30 and s4 < 55) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,6,", "shanghai,6,", + }; + tableResultSetEqualTest( + "select province,count_if(s5) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "select count_if(device_id) from table1", + "701: Aggregate functions [count_if] should only have one boolean expression as argument", + DATABASE_NAME); + + tableAssertTestFail( + "select count_if(s5, device_id != 'd01') from table1", + "701: Aggregate functions [count_if] should only have one boolean expression as argument", + DATABASE_NAME); + } + + @Test + public void avgTest() { + String[] expectedHeader = new String[] {"device_id", "_col1"}; + String[] retArray = + new String[] { + "d01,45.0,", + }; + tableResultSetEqualTest( + "select device_id, avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,55.0,", + }; + tableResultSetEqualTest( + "select device_id, avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, avg(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, avg(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,42500.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,38500.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,42500.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,38500.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,42500.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,38500.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,42500.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,38500.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, avg(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,44.5,", + "beijing,beijing,haidian,44.5,", + "shanghai,shanghai,huangpu,44.5,", + "shanghai,shanghai,pudong,44.5,", + }; + tableResultSetEqualTest( + "select province,city,region,avg(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,44.5,", "shanghai,shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,city,avg(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,44.5,", "shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,avg(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"44.5,"}; + tableResultSetEqualTest("select avg(s4) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void sumTest() { + String[] expectedHeader = new String[] {"device_id", "_col1"}; + String[] retArray = + new String[] { + "d01,90.0,", + }; + tableResultSetEqualTest( + "select device_id, sum(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,55.0,", + }; + tableResultSetEqualTest( + "select device_id, sum(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, sum(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, sum(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,85000.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,77000.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,85000.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,77000.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,85000.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,77000.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,85000.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,77000.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, sum(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,267.0,", + "beijing,beijing,haidian,267.0,", + "shanghai,shanghai,huangpu,267.0,", + "shanghai,shanghai,pudong,267.0,", + }; + tableResultSetEqualTest( + "select province,city,region,sum(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,534.0,", "shanghai,shanghai,534.0,", + }; + tableResultSetEqualTest( + "select province,city,sum(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,534.0,", "shanghai,534.0,", + }; + tableResultSetEqualTest( + "select province,sum(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"908.0,"}; + tableResultSetEqualTest("select sum(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void minTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,35.0,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select device_id, min(time),min(s4), min(s9), min(s10) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:40.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, min(time),min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30.0,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, min(time),min(s3) from table1 where device_id = 'd01' group by 1, 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,min(time),min(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,min(time),min(s1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,35000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,35000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,35000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,35000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,40000,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,min(time),min(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:30.000Z,35.0,", + "beijing,beijing,haidian,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,pudong,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,city,region,min(time),min(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2", "_col3"}; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,city,min(time),min(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1", "_col2"}; + retArray = + new String[] { + "beijing,2024-09-24T06:15:30.000Z,35.0,", "shanghai,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,min(time),min(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"2024-09-24T06:15:30.000Z,30.0,"}; + tableResultSetEqualTest( + "select min(time),min(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void minByTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:35.000Z,35.0,", + }; + tableResultSetEqualTest( + "select device_id, min_by(time, s4), min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, min_by(time, s4), min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,30.0,", + "d01,2024-09-24T06:15:35.000Z,35.0,", + "d01,2024-09-24T06:15:40.000Z,40.0,", + "d01,null,null,", + "d01,null,null,", + }; + + tableResultSetEqualTest( + "select device_id, min_by(time, s3), min(s3) from table1 where device_id = 'd01' group by date_bin(5s, time), 1 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),min_by(time, s4), min(s4) from table1 group by 1,2,3,4,date_bin(5s, time) order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),min_by(time, s1), min(s1) from table1 group by date_bin(5s, time),1,2,3,4 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,40000,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,min_by(time, s2), min(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"40,"}; + tableResultSetEqualTest( + "select min_by(s1, s10) from table1 where s1=40", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select device_id,max(time),max(s4),max(s9),max(s10) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id,max(time),max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30.0,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, max(time),max(s3) from table1 where device_id = 'd01' group by 1, 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,40.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,46.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,40.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,46.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,40.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,46.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,40.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,46.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,max(time),max(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2", "_col3"}; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,max(time),max(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1", "_col2"}; + retArray = + new String[] { + "beijing,2024-09-24T06:15:55.000Z,55.0,", "shanghai,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,max(time),max(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"2024-09-24T06:15:55.000Z,51.0,"}; + tableResultSetEqualTest( + "select max(time),max(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxByTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, max_by(time, s4), max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, max_by(time, s4), max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "d01,2024-09-24T06:15:50.000Z,null,null,", + "d01,2024-09-24T06:15:55.000Z,null,null,", + }; + + tableResultSetEqualTest( + "select device_id, date_bin(5s, time), max_by(time, s3), max(s3) from table1 where device_id = 'd01' group by date_bin(5s, time), 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),max_by(time, s4), max(s4) from table1 group by 1,2,3,4,date_bin(5s, time) order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),max_by(time, s1), max(s1) from table1 group by date_bin(5s, time),1,2,3,4 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,40000,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,max_by(time, s2), max(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"40,"}; + tableResultSetEqualTest( + "select max_by(s1, s10) from table1 where s1=40", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void firstTest() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "device_id", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_chaoyang_red_B_d10_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_chaoyang_yellow_A_d11_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_chaoyang_yellow_A_d11_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_chaoyang_yellow_A_d11_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_chaoyang_yellow_A_d11_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_haidian_red_A_d13_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_haidian_red_A_d13_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_haidian_red_B_d14_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_haidian_yellow_A_d15_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_haidian_yellow_A_d15_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_haidian_yellow_A_d15_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_haidian_yellow_A_d15_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_haidian_yellow_B_d16_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_huangpu_red_B_d02_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_huangpu_yellow_A_d03_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_huangpu_yellow_A_d03_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_huangpu_yellow_A_d03_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_pudong_red_A_d05_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_pudong_red_A_d05_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_pudong_red_B_d06_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_pudong_yellow_A_d07_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_pudong_yellow_A_d07_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_pudong_yellow_A_d07_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_pudong_yellow_B_d08_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_pudong_yellow_B_d08_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,beijing_haidian_yellow_A_d15_41,beijing_haidian_yellow_A_d15_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,shanghai_pudong_yellow_A_d07_41,shanghai_pudong_yellow_A_d07_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:35.000Z,35000,", + }; + tableResultSetEqualTest( + "select first_by(time,s2),first(s2) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = + new String[] { + "null,2024-09-24T06:15:30.000Z,", + }; + tableResultSetEqualTest( + "select first_by(s2,time),first(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,null,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,null,null,", + "2024-09-24T06:15:50.000Z,d01,null,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, first_by(time, s4), first(s4) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,null,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first_by(time,s3), first(s3) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,beijing_chaoyang_red_A_d09_35,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,beijing_chaoyang_red_B_d10_36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,beijing_chaoyang_yellow_A_d11_36,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,beijing_chaoyang_yellow_B_d12_30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,beijing_haidian_red_A_d13_35,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,beijing_haidian_red_B_d14_36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,beijing_haidian_yellow_A_d15_36,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,beijing_haidian_yellow_B_d16_30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,shanghai_huangpu_red_A_d01_35,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,shanghai_huangpu_red_B_d02_36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,shanghai_huangpu_yellow_A_d03_36,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,shanghai_huangpu_yellow_B_d04_30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,shanghai_pudong_red_A_d05_35,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,shanghai_pudong_red_B_d06_36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,shanghai_pudong_yellow_A_d07_36,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,shanghai_pudong_yellow_B_d08_30,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,first_by(time,s7),first(s7) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"city", "region", "device_id", "_col3"}; + retArray = + new String[] { + "beijing,chaoyang,d09,null,", + "beijing,chaoyang,d10,true,", + "beijing,chaoyang,d11,null,", + "beijing,chaoyang,d12,true,", + "beijing,haidian,d13,null,", + "beijing,haidian,d14,true,", + "beijing,haidian,d15,null,", + "beijing,haidian,d16,true,", + "shanghai,huangpu,d01,null,", + "shanghai,huangpu,d02,true,", + "shanghai,huangpu,d03,null,", + "shanghai,huangpu,d04,true,", + "shanghai,pudong,d05,null,", + "shanghai,pudong,d06,true,", + "shanghai,pudong,d07,null,", + "shanghai,pudong,d08,true,", + }; + tableResultSetEqualTest( + "select city,region,device_id,first_by(s5,time,time) from table1 group by city,region,device_id order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastTest() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "device_id", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_chaoyang_red_B_d10_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_chaoyang_yellow_A_d11_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_chaoyang_yellow_A_d11_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_chaoyang_yellow_A_d11_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_chaoyang_yellow_A_d11_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_haidian_red_A_d13_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_haidian_red_A_d13_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_haidian_red_B_d14_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_haidian_yellow_A_d15_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_haidian_yellow_A_d15_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_haidian_yellow_A_d15_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_haidian_yellow_A_d15_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_haidian_yellow_B_d16_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_huangpu_red_B_d02_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_huangpu_yellow_A_d03_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_huangpu_yellow_A_d03_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_huangpu_yellow_A_d03_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_pudong_red_A_d05_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_pudong_red_A_d05_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_pudong_red_B_d06_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_pudong_yellow_A_d07_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_pudong_yellow_A_d07_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_pudong_yellow_A_d07_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_pudong_yellow_B_d08_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_pudong_yellow_B_d08_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,beijing_haidian_yellow_A_d15_51,beijing_haidian_yellow_A_d15_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,shanghai_pudong_yellow_A_d07_51,shanghai_pudong_yellow_A_d07_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:50.000Z,50000,", + }; + repeatTest( + "select last_by(time,s2),last(s2) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = + new String[] { + "null,2024-09-24T06:15:55.000Z,", + }; + repeatTest( + "select last_by(s2, time),last(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,null,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,null,null,", + "2024-09-24T06:15:50.000Z,d01,null,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55.0,", + }; + repeatTest( + "select date_bin(5s, time), device_id, last_by(time, s4), last(s4) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,null,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first_by(time,s3), first(s3) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,0xcafebabe55,", + }; + repeatTest( + "select province,city,region,device_id,last_by(time,s8),last(s8) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"city", "region", "device_id", "_col3"}; + retArray = + new String[] { + "beijing,chaoyang,d09,null,", + "beijing,chaoyang,d10,null,", + "beijing,chaoyang,d11,null,", + "beijing,chaoyang,d12,null,", + "beijing,haidian,d13,null,", + "beijing,haidian,d14,null,", + "beijing,haidian,d15,null,", + "beijing,haidian,d16,null,", + "shanghai,huangpu,d01,null,", + "shanghai,huangpu,d02,null,", + "shanghai,huangpu,d03,null,", + "shanghai,huangpu,d04,null,", + "shanghai,pudong,d05,null,", + "shanghai,pudong,d06,null,", + "shanghai,pudong,d07,null,", + "shanghai,pudong,d08,null,", + }; + repeatTest( + "select city,region,device_id,last_by(s5,time) from table1 group by city,region,device_id order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + } + + @Test + public void extremeTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3"}; + String[] retArray = new String[] {"d01,55,50000,55.0,"}; + + tableResultSetEqualTest( + "select device_id, extreme(s1), extreme(s2), extreme(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {"d01,40.0,"}; + tableResultSetEqualTest( + "select device_id, extreme(s3) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id,extreme(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,extreme(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,extreme(s1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,50000,", + "beijing,beijing,chaoyang,d10,50000,", + "beijing,beijing,chaoyang,d11,46000,", + "beijing,beijing,chaoyang,d12,40000,", + "beijing,beijing,haidian,d13,50000,", + "beijing,beijing,haidian,d14,50000,", + "beijing,beijing,haidian,d15,46000,", + "beijing,beijing,haidian,d16,40000,", + "shanghai,shanghai,huangpu,d01,50000,", + "shanghai,shanghai,huangpu,d02,50000,", + "shanghai,shanghai,huangpu,d03,46000,", + "shanghai,shanghai,huangpu,d04,40000,", + "shanghai,shanghai,pudong,d05,50000,", + "shanghai,shanghai,pudong,d06,50000,", + "shanghai,shanghai,pudong,d07,46000,", + "shanghai,shanghai,pudong,d08,40000,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,extreme(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,55.0,", + "beijing,beijing,haidian,55.0,", + "shanghai,shanghai,huangpu,55.0,", + "shanghai,shanghai,pudong,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,extreme(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,55.0,", "shanghai,shanghai,55.0,", + }; + tableResultSetEqualTest( + "select province,city,extreme(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,55.0,", "shanghai,55.0,", + }; + tableResultSetEqualTest( + "select province,extreme(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "51.0,", + }; + tableResultSetEqualTest( + "select extreme(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void groupByValueTest() { + + String[] expectedHeader = new String[] {"s1", "_col1"}; + String[] retArray = + new String[] { + "30,4,", "36,8,", "40,8,", "41,4,", "55,8,", "null,32,", + }; + tableResultSetEqualTest( + "select s1, count(*) from table1 group by s1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s1", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30,1,", + "beijing,beijing,chaoyang,d09,40,1,", + "beijing,beijing,chaoyang,d09,55,1,", + "beijing,beijing,chaoyang,d09,null,2,", + "beijing,beijing,chaoyang,d10,36,1,", + "beijing,beijing,chaoyang,d10,40,1,", + "beijing,beijing,chaoyang,d10,null,1,", + "beijing,beijing,chaoyang,d11,36,1,", + "beijing,beijing,chaoyang,d11,41,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,30,1,", + "beijing,beijing,haidian,d13,40,1,", + "beijing,beijing,haidian,d13,55,1,", + "beijing,beijing,haidian,d13,null,2,", + "beijing,beijing,haidian,d14,36,1,", + "beijing,beijing,haidian,d14,40,1,", + "beijing,beijing,haidian,d14,null,1,", + "beijing,beijing,haidian,d15,36,1,", + "beijing,beijing,haidian,d15,41,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,30,1,", + "shanghai,shanghai,huangpu,d01,40,1,", + "shanghai,shanghai,huangpu,d01,55,1,", + "shanghai,shanghai,huangpu,d01,null,2,", + "shanghai,shanghai,huangpu,d02,36,1,", + "shanghai,shanghai,huangpu,d02,40,1,", + "shanghai,shanghai,huangpu,d02,null,1,", + "shanghai,shanghai,huangpu,d03,36,1,", + "shanghai,shanghai,huangpu,d03,41,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,30,1,", + "shanghai,shanghai,pudong,d05,40,1,", + "shanghai,shanghai,pudong,d05,55,1,", + "shanghai,shanghai,pudong,d05,null,2,", + "shanghai,shanghai,pudong,d06,36,1,", + "shanghai,shanghai,pudong,d06,40,1,", + "shanghai,shanghai,pudong,d06,null,1,", + "shanghai,shanghai,pudong,d07,36,1,", + "shanghai,shanghai,pudong,d07,41,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s1,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s2", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,35000,1,", + "beijing,beijing,chaoyang,d09,50000,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,50000,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,31000,1,", + "beijing,beijing,chaoyang,d11,46000,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,40000,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,35000,1,", + "beijing,beijing,haidian,d13,50000,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,50000,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,31000,1,", + "beijing,beijing,haidian,d15,46000,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,40000,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,35000,1,", + "shanghai,shanghai,huangpu,d01,50000,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,50000,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,31000,1,", + "shanghai,shanghai,huangpu,d03,46000,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,40000,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,35000,1,", + "shanghai,shanghai,pudong,d05,50000,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,50000,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,31000,1,", + "shanghai,shanghai,pudong,d07,46000,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,40000,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s2,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s3", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30.0,1,", + "beijing,beijing,chaoyang,d09,35.0,1,", + "beijing,beijing,chaoyang,d09,40.0,1,", + "beijing,beijing,chaoyang,d09,null,2,", + "beijing,beijing,chaoyang,d10,null,3,", + "beijing,beijing,chaoyang,d11,41.0,1,", + "beijing,beijing,chaoyang,d11,51.0,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,30.0,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,30.0,1,", + "beijing,beijing,haidian,d13,35.0,1,", + "beijing,beijing,haidian,d13,40.0,1,", + "beijing,beijing,haidian,d13,null,2,", + "beijing,beijing,haidian,d14,null,3,", + "beijing,beijing,haidian,d15,41.0,1,", + "beijing,beijing,haidian,d15,51.0,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,30.0,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,30.0,1,", + "shanghai,shanghai,huangpu,d01,35.0,1,", + "shanghai,shanghai,huangpu,d01,40.0,1,", + "shanghai,shanghai,huangpu,d01,null,2,", + "shanghai,shanghai,huangpu,d02,null,3,", + "shanghai,shanghai,huangpu,d03,41.0,1,", + "shanghai,shanghai,huangpu,d03,51.0,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,30.0,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,30.0,1,", + "shanghai,shanghai,pudong,d05,35.0,1,", + "shanghai,shanghai,pudong,d05,40.0,1,", + "shanghai,shanghai,pudong,d05,null,2,", + "shanghai,shanghai,pudong,d06,null,3,", + "shanghai,shanghai,pudong,d07,41.0,1,", + "shanghai,shanghai,pudong,d07,51.0,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,30.0,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s3,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,35.0,1,", + "beijing,beijing,chaoyang,d09,55.0,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,40.0,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,36.0,1,", + "beijing,beijing,chaoyang,d11,46.0,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,55.0,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,35.0,1,", + "beijing,beijing,haidian,d13,55.0,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,40.0,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,36.0,1,", + "beijing,beijing,haidian,d15,46.0,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,55.0,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,35.0,1,", + "shanghai,shanghai,huangpu,d01,55.0,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,40.0,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,36.0,1,", + "shanghai,shanghai,huangpu,d03,46.0,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,55.0,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,35.0,1,", + "shanghai,shanghai,pudong,d05,55.0,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,40.0,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,36.0,1,", + "shanghai,shanghai,pudong,d07,46.0,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,55.0,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s4,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s5", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,false,1,", + "beijing,beijing,chaoyang,d09,true,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,true,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,false,1,", + "beijing,beijing,chaoyang,d11,null,4,", + "beijing,beijing,chaoyang,d12,true,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,false,1,", + "beijing,beijing,haidian,d13,true,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,true,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,false,1,", + "beijing,beijing,haidian,d15,null,4,", + "beijing,beijing,haidian,d16,true,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,false,1,", + "shanghai,shanghai,huangpu,d01,true,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,true,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,false,1,", + "shanghai,shanghai,huangpu,d03,null,4,", + "shanghai,shanghai,huangpu,d04,true,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,false,1,", + "shanghai,shanghai,pudong,d05,true,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,true,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,false,1,", + "shanghai,shanghai,pudong,d07,null,4,", + "shanghai,shanghai,pudong,d08,true,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s5,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s6", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_30,1,", + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_35,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_36,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_41,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_51,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,beijing_chaoyang_yellow_B_d12_55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_30,1,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_35,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_36,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_41,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_51,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,beijing_haidian_yellow_B_d16_55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_30,1,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_35,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_36,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_41,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_51,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,shanghai_huangpu_yellow_B_d04_55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_30,1,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_35,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_36,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_41,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_51,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,shanghai_pudong_yellow_B_d08_55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s6,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s7", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_35,1,", + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_40,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_36,1,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_40,1,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_50,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_36,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_46,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,beijing_chaoyang_yellow_B_d12_30,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_35,1,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_40,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_36,1,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_40,1,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_50,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_36,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_46,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,beijing_haidian_yellow_B_d16_30,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_35,1,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_40,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_36,1,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_40,1,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_50,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_36,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_46,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,shanghai_huangpu_yellow_B_d04_30,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_35,1,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_40,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_36,1,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_40,1,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_50,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_36,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_46,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,shanghai_pudong_yellow_B_d08_30,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s7,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,device_id,s7,count(1) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s8", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,0xcafebabe30,1,", + "beijing,beijing,chaoyang,d09,0xcafebabe55,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,0xcafebabe50,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,0xcafebabe31,1,", + "beijing,beijing,chaoyang,d11,0xcafebabe41,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,0xcafebabe55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,0xcafebabe30,1,", + "beijing,beijing,haidian,d13,0xcafebabe55,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,0xcafebabe50,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,0xcafebabe31,1,", + "beijing,beijing,haidian,d15,0xcafebabe41,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,0xcafebabe55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,0xcafebabe30,1,", + "shanghai,shanghai,huangpu,d01,0xcafebabe55,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,0xcafebabe50,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,0xcafebabe31,1,", + "shanghai,shanghai,huangpu,d03,0xcafebabe41,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,0xcafebabe55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,0xcafebabe30,1,", + "shanghai,shanghai,pudong,d05,0xcafebabe55,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,0xcafebabe50,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,0xcafebabe31,1,", + "shanghai,shanghai,pudong,d07,0xcafebabe41,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,0xcafebabe55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s8,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s9", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s9,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s10", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24,2,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,2024-09-24,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,2024-09-24,1,", + "beijing,beijing,chaoyang,d11,null,4,", + "beijing,beijing,chaoyang,d12,2024-09-24,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,2024-09-24,2,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,2024-09-24,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,2024-09-24,1,", + "beijing,beijing,haidian,d15,null,4,", + "beijing,beijing,haidian,d16,2024-09-24,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,2024-09-24,2,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,2024-09-24,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,2024-09-24,1,", + "shanghai,shanghai,huangpu,d03,null,4,", + "shanghai,shanghai,huangpu,d04,2024-09-24,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,2024-09-24,2,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,2024-09-24,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,2024-09-24,1,", + "shanghai,shanghai,pudong,d07,null,4,", + "shanghai,shanghai,pudong,d08,2024-09-24,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s10,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastQueryTest() { + + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id='d01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d09,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d12,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id,date_bin(5s,time),last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by province,city,region,device_id,date_bin(5s,time) order by device_id,date_bin(5s,time)", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s,time),last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd08', 'd12', 'd13') group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd05', 'd08', 'd09', 'd12', 'd13') group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "beijing,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd05', 'd08', 'd09', 'd12', 'd13') group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d02,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "d03,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d05,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d06,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "d07,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id, last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where city = 'shanghai' group by province,city,region,device_id order by device_id limit 7", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void specialCasesTest() { + String[] expectedHeader = new String[] {"device_id"}; + String[] retArray = + new String[] { + "d01,", "d02,", "d03,", "d04,", "d05,", "d06,", "d07,", "d08,", "d09,", "d10,", "d11,", + "d12,", "d13,", "d14,", "d15,", "d16,", + }; + tableResultSetEqualTest( + "SELECT device_id FROM table1 GROUP BY device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,", + "2024-09-24T06:15:31.000Z,", + "2024-09-24T06:15:35.000Z,", + "2024-09-24T06:15:36.000Z,", + "2024-09-24T06:15:40.000Z,", + "2024-09-24T06:15:41.000Z,", + "2024-09-24T06:15:46.000Z,", + "2024-09-24T06:15:50.000Z,", + "2024-09-24T06:15:51.000Z,", + "2024-09-24T06:15:55.000Z," + }; + tableResultSetEqualTest( + "select time from table1 group by time order by time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void modeTest() { + // AggTableScan + Agg mixed test + String[] expectedHeader = buildHeaders(10); + String[] retArray = + new String[] { + "null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + }; + tableResultSetEqualTest( + "select mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6),mode(s7),mode(s8),mode(s9),mode(s10) from table1 group by city", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void approxCountDistinctTest() { + String[] expectedHeader = buildHeaders(15); + String[] retArray = new String[] {"10,2,2,4,16,5,5,5,5,2,24,32,5,10,1,"}; + tableResultSetEqualTest( + "select approx_count_distinct(time), approx_count_distinct(province), approx_count_distinct(city), approx_count_distinct(region), approx_count_distinct(device_id), approx_count_distinct(s1), approx_count_distinct(s2), approx_count_distinct(s3), approx_count_distinct(s4), approx_count_distinct(s5), approx_count_distinct(s6), approx_count_distinct(s7), approx_count_distinct(s8), approx_count_distinct(s9), approx_count_distinct(s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select approx_count_distinct(time, 0.02), approx_count_distinct(province, 0.02), approx_count_distinct(city, 0.02), approx_count_distinct(region, 0.02), approx_count_distinct(device_id, 0.02), approx_count_distinct(s1, 0.02), approx_count_distinct(s2, 0.02), approx_count_distinct(s3, 0.02), approx_count_distinct(s4, 0.02), approx_count_distinct(s5, 0.02), approx_count_distinct(s6, 0.02), approx_count_distinct(s7, 0.02), approx_count_distinct(s8, 0.02), approx_count_distinct(s9, 0.02), approx_count_distinct(s10, 0.02) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,2,2,", + "2024-09-24T06:15:31.000Z,beijing,0,0,", + "2024-09-24T06:15:35.000Z,beijing,2,2,", + "2024-09-24T06:15:36.000Z,beijing,2,4,", + "2024-09-24T06:15:40.000Z,beijing,0,4,", + "2024-09-24T06:15:41.000Z,beijing,2,0,", + "2024-09-24T06:15:46.000Z,beijing,0,2,", + "2024-09-24T06:15:50.000Z,beijing,0,2,", + "2024-09-24T06:15:51.000Z,beijing,2,0,", + "2024-09-24T06:15:55.000Z,beijing,2,0,", + "2024-09-24T06:15:30.000Z,shanghai,2,2,", + "2024-09-24T06:15:31.000Z,shanghai,0,0,", + "2024-09-24T06:15:35.000Z,shanghai,2,2,", + "2024-09-24T06:15:36.000Z,shanghai,2,4,", + "2024-09-24T06:15:40.000Z,shanghai,0,4,", + "2024-09-24T06:15:41.000Z,shanghai,2,0,", + "2024-09-24T06:15:46.000Z,shanghai,0,2,", + "2024-09-24T06:15:50.000Z,shanghai,0,2,", + "2024-09-24T06:15:51.000Z,shanghai,2,0,", + "2024-09-24T06:15:55.000Z,shanghai,2,0,", + }; + + tableResultSetEqualTest( + "select time,province,approx_count_distinct(s6),approx_count_distinct(s7) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2", "_col3"}, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,province,approx_count_distinct(s6,0.02),approx_count_distinct(s7,0.02) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2", "_col3"}, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest() { + tableAssertTestFail( + "select avg() from table1", + "701: Aggregate functions [avg] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select sum() from table1", + "701: Aggregate functions [sum] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select extreme() from table1", + "701: Aggregate functions [extreme] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select first() from table1", + "701: Aggregate functions [first] should only have one or two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select first_by() from table1", + "701: Aggregate functions [first_by] should only have two or three arguments", + DATABASE_NAME); + tableAssertTestFail( + "select last() from table1", + "701: Aggregate functions [last] should only have one or two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select last_by() from table1", + "701: Aggregate functions [last_by] should only have two or three arguments", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct() from table1", + "701: Aggregate functions [approx_count_distinct] should only have two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 0.3) from table1", + "750: Max Standard Error must be in [0.0040625, 0.26]: 0.3", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 0.3) from table1", + "750: Max Standard Error must be in [0.0040625, 0.26]: 0.3", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 'test') from table1", + "701: Second argument of Aggregate functions [approx_count_distinct] should be numberic type and do not use expression", + DATABASE_NAME); + } + + // ================================================================== + // ===================== Select Distinct Test ======================= + // ================================================================== + + // Select distinct is a special kind of aggregate query in actual, so we put ITs here to reuse the + // test data. + + @Test + public void simpleTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = new String[] {"30,", "36,", "40,", "41,", "55,", "null,"}; + tableResultSetEqualTest( + "select distinct s1 from table1 order by s1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"region", "s1"}; + retArray = + new String[] { + "chaoyang,30,", + "chaoyang,36,", + "chaoyang,40,", + "chaoyang,41,", + "chaoyang,55,", + "chaoyang,null,", + "haidian,30,", + "haidian,36,", + "haidian,40,", + "haidian,41,", + "haidian,55,", + "haidian,null,", + "huangpu,30,", + "huangpu,36,", + "huangpu,40,", + "huangpu,41,", + "huangpu,55,", + "huangpu,null,", + "pudong,30,", + "pudong,36,", + "pudong,40,", + "pudong,41,", + "pudong,55,", + "pudong,null," + }; + tableResultSetEqualTest( + "select distinct region, s1 from table1 order by region, s1", + expectedHeader, + retArray, + DATABASE_NAME); + + // show all devices + expectedHeader = new String[] {"province", "city", "region", "device_id"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,", + "beijing,beijing,chaoyang,d10,", + "beijing,beijing,chaoyang,d11,", + "beijing,beijing,chaoyang,d12,", + "beijing,beijing,haidian,d13,", + "beijing,beijing,haidian,d14,", + "beijing,beijing,haidian,d15,", + "beijing,beijing,haidian,d16,", + "shanghai,shanghai,huangpu,d01,", + "shanghai,shanghai,huangpu,d02,", + "shanghai,shanghai,huangpu,d03,", + "shanghai,shanghai,huangpu,d04,", + "shanghai,shanghai,pudong,d05,", + "shanghai,shanghai,pudong,d06,", + "shanghai,shanghai,pudong,d07,", + "shanghai,shanghai,pudong,d08,", + }; + tableResultSetEqualTest( + "select distinct province,city,region,device_id from table1 order by province,city,region,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,"}; + tableResultSetEqualTest( + "select distinct s1 < 0 from table1 where s1 is not null", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void withGroupByTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = new String[] {"30,", "36,", "40,", "41,", "55,", "null,"}; + tableResultSetEqualTest( + "select distinct s1 from table1 group by s1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct s1 from table1 group by s1,s2 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"30.0,", "36.0,", "40.0,", "41.0,", "55.0,", "null,"}; + tableResultSetEqualTest( + "select distinct avg(s1) from table1 group by s1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct avg(s1) from table1 group by s1,s2 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = new String[] {"4,", "8,", "32,"}; + tableResultSetEqualTest( + "select distinct count(*) from table1 group by s1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = new String[] {"4,", "8,"}; + tableResultSetEqualTest( + "select distinct count(*) from table1 group by s1, s2 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest1() { + tableAssertTestFail( + "select distinct s1 from table1 order by s2", + "701: For SELECT DISTINCT, ORDER BY expressions must appear in select list", + DATABASE_NAME); + } + + // ================================================================== + // ================== Agg-Function Distinct Test ==================== + // ================================================================== + @Test + public void countDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "2,2,4,16,5,5,5,5,2,24,32,5,10,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select count(distinct province), count(distinct city), count(distinct region), count(distinct device_id), count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,5,5,5,2,6,8,5,10,1,", + "beijing,beijing,haidian,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,huangpu,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,pudong,5,5,5,5,2,6,8,5,10,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,5,5,5,5,2,6,8,5,10,1,", + "haidian,5,5,5,5,2,6,8,5,10,1,", + "huangpu,5,5,5,5,2,6,8,5,10,1,", + "pudong,5,5,5,5,2,6,8,5,10,1," + }; + tableResultSetEqualTest( + "select region, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void countIfDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "1,1,1,1,0,1,1,1,1,1,1,1,1,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select count_if(distinct province = 'shanghai'), count_if(distinct city = 'shanghai'), count_if(distinct region= 'huangpu'), count_if(distinct device_id = 'd03'), count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,0,1,1,1,1,1,1,1,1,1,", + "beijing,beijing,haidian,0,1,1,1,1,1,1,1,1,1,", + "shanghai,shanghai,huangpu,0,1,1,1,1,1,1,1,1,1,", + "shanghai,shanghai,pudong,0,1,1,1,1,1,1,1,1,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,0,1,1,1,1,1,1,1,1,1,", + "haidian,0,1,1,1,1,1,1,1,1,1,", + "huangpu,0,1,1,1,1,1,1,1,1,1,", + "pudong,0,1,1,1,1,1,1,1,1,1," + }; + tableResultSetEqualTest( + "select region, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void avgDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "40.4,40400.0,39.4,42.4,", + }; + // global Aggregation + tableResultSetEqualTest( + "select avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,40.4,40400.0,39.4,42.4,", + "beijing,beijing,haidian,40.4,40400.0,39.4,42.4,", + "shanghai,shanghai,huangpu,40.4,40400.0,39.4,42.4,", + "shanghai,shanghai,pudong,40.4,40400.0,39.4,42.4," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,40.4,40400.0,39.4,42.4,", + "haidian,40.4,40400.0,39.4,42.4,", + "huangpu,40.4,40400.0,39.4,42.4,", + "pudong,40.4,40400.0,39.4,42.4," + }; + tableResultSetEqualTest( + "select region, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sumDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "202.0,202000.0,197.0,212.0,", + }; + // global Aggregation + tableResultSetEqualTest( + "select sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,202.0,202000.0,197.0,212.0,", + "beijing,beijing,haidian,202.0,202000.0,197.0,212.0,", + "shanghai,shanghai,huangpu,202.0,202000.0,197.0,212.0,", + "shanghai,shanghai,pudong,202.0,202000.0,197.0,212.0," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,202.0,202000.0,197.0,212.0,", + "haidian,202.0,202000.0,197.0,212.0,", + "huangpu,202.0,202000.0,197.0,212.0,", + "pudong,202.0,202000.0,197.0,212.0," + }; + tableResultSetEqualTest( + "select region, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "beijing,beijing,chaoyang,d01,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select min(distinct province), min(distinct city), min(distinct region), min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,30,31000,30.0,35.0,false,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,30,31000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,30,31000,30.0,35.0,false,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + retArray = + new String[] { + "chaoyang,d09,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "haidian,d13,30,31000,30.0,35.0,false,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "huangpu,d01,30,31000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "pudong,d05,30,31000,30.0,35.0,false,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + retArray = + new String[] { + "d03,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "d11,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z," + }; + + // test GroupByDistinctAccumulator + tableResultSetEqualTest( + "select device_id, min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "shanghai,shanghai,pudong,d16,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select max(distinct province), max(distinct city), max(distinct region), max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d12,55,50000,51.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,55,50000,51.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,55,50000,51.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + retArray = + new String[] { + "chaoyang,d12,55,50000,51.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "haidian,d16,55,50000,51.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "huangpu,d04,55,50000,51.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "pudong,d08,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + retArray = + new String[] { + "d03,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "d11,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z," + }; + + // test GroupByDistinctAccumulator + tableResultSetEqualTest( + "select device_id, max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "huangpu,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + // global Aggregation + tableResultSetEqualTest( + "select first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "huangpu,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select region, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "huangpu,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + // global Aggregation + tableResultSetEqualTest( + "select last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), last_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "huangpu,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select region, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extremeDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "55,50000,51.0,55.0,", + }; + // global Aggregation + tableResultSetEqualTest( + "select extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,55,50000,51.0,55.0,", + "beijing,beijing,haidian,55,50000,51.0,55.0,", + "shanghai,shanghai,huangpu,55,50000,51.0,55.0,", + "shanghai,shanghai,pudong,55,50000,51.0,55.0," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,55,50000,51.0,55.0,", + "haidian,55,50000,51.0,55.0,", + "huangpu,55,50000,51.0,55.0,", + "pudong,55,50000,51.0,55.0," + }; + tableResultSetEqualTest( + "select region, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void varianceDistinctTest() { + // The addInput logic of all variance functions are the same, so test any one function is ok. + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "68.2,4.824E7,49.0,54.6,", + }; + tableResultSetEqualTest( + "select round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,68.2,4.824E7,49.0,54.6,", + "beijing,beijing,haidian,68.2,4.824E7,49.0,54.6,", + "shanghai,shanghai,huangpu,68.2,4.824E7,49.0,54.6,", + "shanghai,shanghai,pudong,68.2,4.824E7,49.0,54.6," + }; + tableResultSetEqualTest( + "select province,city,region, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,68.2,4.824E7,49.0,54.6,", + "haidian,68.2,4.824E7,49.0,54.6,", + "huangpu,68.2,4.824E7,49.0,54.6,", + "pudong,68.2,4.824E7,49.0,54.6," + }; + tableResultSetEqualTest( + "select region, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void mixedTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "32,5,", + }; + + tableResultSetEqualTest( + "select count(s1), count(distinct s1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,3,", + "beijing,beijing,chaoyang,d10,2,2,", + "beijing,beijing,chaoyang,d11,2,2,", + "beijing,beijing,chaoyang,d12,1,1,", + "beijing,beijing,haidian,d13,3,3,", + "beijing,beijing,haidian,d14,2,2,", + "beijing,beijing,haidian,d15,2,2,", + "beijing,beijing,haidian,d16,1,1,", + "shanghai,shanghai,huangpu,d01,3,3,", + "shanghai,shanghai,huangpu,d02,2,2,", + "shanghai,shanghai,huangpu,d03,2,2,", + "shanghai,shanghai,huangpu,d04,1,1,", + "shanghai,shanghai,pudong,d05,3,3,", + "shanghai,shanghai,pudong,d06,2,2,", + "shanghai,shanghai,pudong,d07,2,2,", + "shanghai,shanghai,pudong,d08,1,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,count(s1), count(distinct s1) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void singleInputDistinctAggregationTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "5,40.4,", + }; + + tableResultSetEqualTest( + "select count(distinct s1), avg(distinct s1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"16,"}; + tableResultSetEqualTest( + "select count(distinct device_id) from table1 group by date_bin(1d,time) order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,40.4,", + "beijing,beijing,haidian,5,40.4,", + "shanghai,shanghai,huangpu,5,40.4,", + "shanghai,shanghai,pudong,5,40.4," + }; + tableResultSetEqualTest( + "select province,city,region,count(distinct s1), avg(distinct s1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest2() { + tableAssertTestFail( + "select count(distinct *) from table1", + "mismatched input '*'. Expecting: ", + DATABASE_NAME); + + String errMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Unsupported expression: Row"; + tableAssertTestFail("select distinct (s1,s2) from table1", errMsg, DATABASE_NAME); + + tableAssertTestFail("select (s1,s2) from table1", errMsg, DATABASE_NAME); + + tableAssertTestFail("select * from table1 where (s1,s2) is not null", errMsg, DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java new file mode 100644 index 0000000000000..dbe22b168f1fd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java @@ -0,0 +1,5054 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.buildHeaders; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.repeatTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableViewAggregationNonStreamIT { + protected static final String TREE_DB_NAME = "root.test"; + protected static final String DATABASE_NAME = "test"; + protected static final String[] createSqls = + new String[] { + "CREATE DATABASE " + TREE_DB_NAME, + "CREATE ALIGNED TIMESERIES root.test.table1.d01(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d02(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d03(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d04(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d05(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d06(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d07(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d08(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d09(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d10(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d11(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d12(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d13(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d14(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d15(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "CREATE ALIGNED TIMESERIES root.test.table1.d16(province STRING, city STRING, region1 STRING, s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT, s7 STRING, s8 BLOB, s9 TIMESTAMP, s10 DATE)", + "INSERT INTO root.test.table1.d01(time,province,city,region1,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu',30,30.0,'shanghai_huangpu_red_A_d01_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.d01(time,province,city,region1,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','huangpu',35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d01(time,province,city,region1,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu',40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d01(time,province,city,region1,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d01(time,province,city,region1,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d02(time,province,city,region1,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu',36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.d02(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu',40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d02(time,province,city,region1,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu',50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.d03(time,province,city,region1,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','huangpu',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.d03(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu',36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d03(time,province,city,region1,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','huangpu',41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.d03(time,province,city,region1,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','huangpu',46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.d03(time,province,city,region1,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','huangpu',51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.d04(time,province,city,region1,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu',30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d04(time,province,city,region1,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d04(time,province,city,region1,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu',55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d05(time,province,city,region1,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong',30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.d05(time,province,city,region1,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','pudong',35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d05(time,province,city,region1,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong',40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d05(time,province,city,region1,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d05(time,province,city,region1,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d06(time,province,city,region1,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong',36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.d06(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong',40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d06(time,province,city,region1,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong',50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.d07(time,province,city,region1,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','pudong',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.d07(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong',36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d07(time,province,city,region1,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','pudong',41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.d07(time,province,city,region1,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','pudong',46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.d07(time,province,city,region1,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','pudong',51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.d08(time,province,city,region1,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong',30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d08(time,province,city,region1,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d08(time,province,city,region1,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong',55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d09(time,province,city,region1,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang',30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.d09(time,province,city,region1,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','chaoyang',35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d09(time,province,city,region1,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang',40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d09(time,province,city,region1,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d09(time,province,city,region1,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d10(time,province,city,region1,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang',36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.d10(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang',40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d10(time,province,city,region1,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang',50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.d11(time,province,city,region1,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','chaoyang',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.d11(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang',36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d11(time,province,city,region1,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','chaoyang',41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.d11(time,province,city,region1,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','chaoyang',46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.d11(time,province,city,region1,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','chaoyang',51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.d12(time,province,city,region1,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang',30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d12(time,province,city,region1,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d12(time,province,city,region1,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang',55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d13(time,province,city,region1,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian',30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO root.test.table1.d13(time,province,city,region1,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','haidian',35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d13(time,province,city,region1,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian',40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d13(time,province,city,region1,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d13(time,province,city,region1,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO root.test.table1.d14(time,province,city,region1,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian',36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO root.test.table1.d14(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian',40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d14(time,province,city,region1,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian',50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO root.test.table1.d15(time,province,city,region1,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','haidian',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO root.test.table1.d15(time,province,city,region1,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian',36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d15(time,province,city,region1,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','haidian',41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO root.test.table1.d15(time,province,city,region1,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','haidian',46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO root.test.table1.d15(time,province,city,region1,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','haidian',51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO root.test.table1.d16(time,province,city,region1,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian',30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO root.test.table1.d16(time,province,city,region1,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO root.test.table1.d16(time,province,city,region1,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian',55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "FLUSH", + }; + + protected static final String[] createTableViewSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW table1(province STRING FIELD, city STRING FIELD, region STRING FIELD FROM region1, device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD) as root.test.table1.**", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(createSqls); + prepareTableData(createTableViewSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ================================================================== + // ==================== Normal Aggregation Test ===================== + // ================================================================== + @Test + public void countTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select count(*) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select count('a') from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count(*) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count(1) from table1 where device_id = 'd01' group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(*) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,1,0,0,1,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,0,1,1,1,0,1,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,1,0,1,0,1,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,0,1,0,0,1,0,0,0,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,1,0,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,1,1,1,0,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,0,1,0,0,0,0,1,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,0,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,1,0,0,1,0,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,1,1,0,1,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,0,1,0,1,0,0,1,0,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,0,0,1,0,1,0,1,0,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,0,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,1,0,0,1,0,1,0,1,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count(s1), count(s2), count(s3), count(s4), count(s5), count(s6), count(s7), count(s8), count(s9), count(s10) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,chaoyang,d10,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,chaoyang,d11,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,chaoyang,d12,1,1,1,1,1,1,1,1,3,1,", + "beijing,beijing,haidian,d13,3,2,3,2,2,2,2,2,5,2,", + "beijing,beijing,haidian,d14,2,1,0,1,1,1,3,1,3,1,", + "beijing,beijing,haidian,d15,2,2,2,2,1,2,2,2,5,1,", + "beijing,beijing,haidian,d16,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,huangpu,d01,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,huangpu,d02,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,huangpu,d03,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,huangpu,d04,1,1,1,1,1,1,1,1,3,1,", + "shanghai,shanghai,pudong,d05,3,2,3,2,2,2,2,2,5,2,", + "shanghai,shanghai,pudong,d06,2,1,0,1,1,1,3,1,3,1,", + "shanghai,shanghai,pudong,d07,2,2,2,2,1,2,2,2,5,1,", + "shanghai,shanghai,pudong,d08,1,1,1,1,1,1,1,1,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, count(s1), count(s2), count(s3), count(s4), count(s5), count(s6), count(s7), count(s8), count(s9), count(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,5,", + "beijing,beijing,chaoyang,d10,3,", + "beijing,beijing,chaoyang,d11,5,", + "beijing,beijing,chaoyang,d12,3,", + "beijing,beijing,haidian,d13,5,", + "beijing,beijing,haidian,d14,3,", + "beijing,beijing,haidian,d15,5,", + "beijing,beijing,haidian,d16,3,", + "shanghai,shanghai,huangpu,d01,5,", + "shanghai,shanghai,huangpu,d02,3,", + "shanghai,shanghai,huangpu,d03,5,", + "shanghai,shanghai,huangpu,d04,3,", + "shanghai,shanghai,pudong,d05,5,", + "shanghai,shanghai,pudong,d06,3,", + "shanghai,shanghai,pudong,d07,5,", + "shanghai,shanghai,pudong,d08,3,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,count(*) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,device_id,count(1) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,16,", + "beijing,beijing,haidian,16,", + "shanghai,shanghai,huangpu,16,", + "shanghai,shanghai,pudong,16,", + }; + tableResultSetEqualTest( + "select province,city,region,count(*) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,count(1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,32,", "shanghai,shanghai,32,", + }; + tableResultSetEqualTest( + "select province,city,count(*) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,count(1) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,32,", "shanghai,32,", + }; + tableResultSetEqualTest( + "select province,count(*) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,count(1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest("select count(*) from table1", expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest("select count(1) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void countIfTest() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = + new String[] { + "5,", + }; + tableResultSetEqualTest( + "select count_if(device_id = 'd01') from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "64,", + }; + tableResultSetEqualTest( + "select count_if(true) from table1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "end_time", "device_id", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d01,1,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d01,1,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d01,1,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d01,1,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d02,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d02,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d02,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d03,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d03,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d03,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d03,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d03,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d04,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d04,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d04,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d05,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d05,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d05,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d05,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d05,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d06,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d06,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d06,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d07,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d07,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d07,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d07,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d07,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d08,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d08,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d08,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d09,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d09,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d09,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d09,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d09,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d10,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d10,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d10,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d11,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d11,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d11,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d11,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d11,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d12,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d12,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d12,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d13,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d13,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d13,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d13,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d13,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d14,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d14,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d14,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d15,0,", + "2024-09-24T06:15:35.000Z,2024-09-24T06:15:40.000Z,d15,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d15,0,", + "2024-09-24T06:15:45.000Z,2024-09-24T06:15:50.000Z,d15,0,", + "2024-09-24T06:15:50.000Z,2024-09-24T06:15:55.000Z,d15,0,", + "2024-09-24T06:15:30.000Z,2024-09-24T06:15:35.000Z,d16,0,", + "2024-09-24T06:15:40.000Z,2024-09-24T06:15:45.000Z,d16,0,", + "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "province", + "city", + "region", + "device_id", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,1,1,1,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,0,0,0,0,1,0,0,1,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,0,0,1,0,0,0,0,0,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,1,0,1,0,0,0,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,0,0,0,0,0,1,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,1,1,1,1,0,1,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,0,0,1,0,1,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,1,0,0,0,0,0,0,1,1,1,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,0,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,0,0,0,0,1,1,1,1,1,0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,1,0,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,1,1,0,0,0,0,0,0,1,0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,0,0,0,1,0,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,0,0,1,0,0,1,0,0,1,0,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,1,1,0,1,0,0,0,1,1,0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,1,0,1,0,0,1,0,1,1,0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,1,0,0,0,1,0,0,1,1,1,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,1,1,0,0,0,0,0,1,1,0,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,0,0,0,0,0,1,0,0,1,0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, count_if(s1 is null), count_if(s2 < 50000), count_if(s3 > 30), count_if(s4 < 55), count_if(s5), count_if(s6 like '%pudong%'), count_if(s7 = 'shanghai_pudong_red_B_d06_36'), count_if(s8 is null), count(s9 is null), count_if(s10 is not null) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2,1,2,1,1,0,0,3,5,2,", + "beijing,beijing,chaoyang,d10,1,0,0,1,1,0,0,2,3,1,", + "beijing,beijing,chaoyang,d11,3,2,2,2,0,0,0,3,5,1,", + "beijing,beijing,chaoyang,d12,2,1,0,0,1,0,0,2,3,1,", + "beijing,beijing,haidian,d13,2,1,2,1,1,0,0,3,5,2,", + "beijing,beijing,haidian,d14,1,0,0,1,1,0,0,2,3,1,", + "beijing,beijing,haidian,d15,3,2,2,2,0,0,0,3,5,1,", + "beijing,beijing,haidian,d16,2,1,0,0,1,0,0,2,3,1,", + "shanghai,shanghai,huangpu,d01,2,1,2,1,1,0,0,3,5,2,", + "shanghai,shanghai,huangpu,d02,1,0,0,1,1,0,0,2,3,1,", + "shanghai,shanghai,huangpu,d03,3,2,2,2,0,0,0,3,5,1,", + "shanghai,shanghai,huangpu,d04,2,1,0,0,1,0,0,2,3,1,", + "shanghai,shanghai,pudong,d05,2,1,2,1,1,2,0,3,5,2,", + "shanghai,shanghai,pudong,d06,1,0,0,1,1,1,1,2,3,1,", + "shanghai,shanghai,pudong,d07,3,2,2,2,0,2,0,3,5,1,", + "shanghai,shanghai,pudong,d08,2,1,0,0,1,1,0,2,3,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, count_if(s1 is null), count_if(s2 < 50000), count_if(s3 > 30), count_if(s4 < 55), count_if(s5), count_if(s6 like '%pudong%'), count_if(s7 = 'shanghai_pudong_red_B_d06_36'), count_if(s8 is null), count(s9 is null), count_if(s10 is not null) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,", + "beijing,beijing,haidian,5,", + "shanghai,shanghai,huangpu,5,", + "shanghai,shanghai,pudong,5,", + }; + tableResultSetEqualTest( + "select province,city,region,count(s3 > 30 and s4 < 55) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,6,", "shanghai,6,", + }; + tableResultSetEqualTest( + "select province,count_if(s5) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableAssertTestFail( + "select count_if(device_id) from table1", + "701: Aggregate functions [count_if] should only have one boolean expression as argument", + DATABASE_NAME); + + tableAssertTestFail( + "select count_if(s5, device_id != 'd01') from table1", + "701: Aggregate functions [count_if] should only have one boolean expression as argument", + DATABASE_NAME); + } + + @Test + public void avgTest() { + String[] expectedHeader = new String[] {"device_id", "_col1"}; + String[] retArray = + new String[] { + "d01,45.0,", + }; + tableResultSetEqualTest( + "select device_id, avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,55.0,", + }; + tableResultSetEqualTest( + "select device_id, avg(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, avg(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, avg(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,42500.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,38500.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,42500.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,38500.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,42500.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,38500.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,42500.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,38500.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, avg(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,44.5,", + "beijing,beijing,haidian,44.5,", + "shanghai,shanghai,huangpu,44.5,", + "shanghai,shanghai,pudong,44.5,", + }; + tableResultSetEqualTest( + "select province,city,region,avg(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,44.5,", "shanghai,shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,city,avg(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,44.5,", "shanghai,44.5,", + }; + tableResultSetEqualTest( + "select province,avg(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"44.5,"}; + tableResultSetEqualTest("select avg(s4) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void sumTest() { + String[] expectedHeader = new String[] {"device_id", "_col1"}; + String[] retArray = + new String[] { + "d01,90.0,", + }; + tableResultSetEqualTest( + "select device_id, sum(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,55.0,", + }; + tableResultSetEqualTest( + "select device_id, sum(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, sum(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id, sum(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,85000.0,", + "beijing,beijing,chaoyang,d10,50000.0,", + "beijing,beijing,chaoyang,d11,77000.0,", + "beijing,beijing,chaoyang,d12,40000.0,", + "beijing,beijing,haidian,d13,85000.0,", + "beijing,beijing,haidian,d14,50000.0,", + "beijing,beijing,haidian,d15,77000.0,", + "beijing,beijing,haidian,d16,40000.0,", + "shanghai,shanghai,huangpu,d01,85000.0,", + "shanghai,shanghai,huangpu,d02,50000.0,", + "shanghai,shanghai,huangpu,d03,77000.0,", + "shanghai,shanghai,huangpu,d04,40000.0,", + "shanghai,shanghai,pudong,d05,85000.0,", + "shanghai,shanghai,pudong,d06,50000.0,", + "shanghai,shanghai,pudong,d07,77000.0,", + "shanghai,shanghai,pudong,d08,40000.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, sum(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,267.0,", + "beijing,beijing,haidian,267.0,", + "shanghai,shanghai,huangpu,267.0,", + "shanghai,shanghai,pudong,267.0,", + }; + tableResultSetEqualTest( + "select province,city,region,sum(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,534.0,", "shanghai,shanghai,534.0,", + }; + tableResultSetEqualTest( + "select province,city,sum(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,534.0,", "shanghai,534.0,", + }; + tableResultSetEqualTest( + "select province,sum(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"908.0,"}; + tableResultSetEqualTest("select sum(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void minTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,35.0,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select device_id, min(time),min(s4), min(s9), min(s10) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:40.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, min(time),min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30.0,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, min(time),min(s3) from table1 where device_id = 'd01' group by 1, 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,min(time),min(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,min(time),min(s1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,35000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,35000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,35000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,35000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,40000,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,min(time),min(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:30.000Z,35.0,", + "beijing,beijing,haidian,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,pudong,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,city,region,min(time),min(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2", "_col3"}; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:30.000Z,35.0,", + "shanghai,shanghai,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,city,min(time),min(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1", "_col2"}; + retArray = + new String[] { + "beijing,2024-09-24T06:15:30.000Z,35.0,", "shanghai,2024-09-24T06:15:30.000Z,35.0,", + }; + tableResultSetEqualTest( + "select province,min(time),min(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"2024-09-24T06:15:30.000Z,30.0,"}; + tableResultSetEqualTest( + "select min(time),min(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void minByTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:35.000Z,35.0,", + }; + tableResultSetEqualTest( + "select device_id, min_by(time, s4), min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, min_by(time, s4), min(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,30.0,", + "d01,2024-09-24T06:15:35.000Z,35.0,", + "d01,2024-09-24T06:15:40.000Z,40.0,", + "d01,null,null,", + "d01,null,null,", + }; + + tableResultSetEqualTest( + "select device_id, min_by(time, s3), min(s3) from table1 where device_id = 'd01' group by date_bin(5s, time), 1 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),min_by(time, s4), min(s4) from table1 group by 1,2,3,4,date_bin(5s, time) order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),min_by(time, s1), min(s1) from table1 group by date_bin(5s, time),1,2,3,4 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,31000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,31000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,40000,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,min_by(time, s2), min(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"40,"}; + tableResultSetEqualTest( + "select min_by(s1, s10) from table1 where s1=40", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3", "_col4"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select device_id,max(time),max(s4),max(s9),max(s10) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id,max(time),max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30.0,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, max(time),max(s3) from table1 where device_id = 'd01' group by 1, 2 order by 2,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"_col0", "province", "city", "region", "device_id", "_col5", "_col6"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,40.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,46.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,40.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,46.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,40.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,46.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,40.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,46.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,max(time),max(s4) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,max(time),max(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2", "_col3"}; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,city,max(time),max(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1", "_col2"}; + retArray = + new String[] { + "beijing,2024-09-24T06:15:55.000Z,55.0,", "shanghai,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select province,max(time),max(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = new String[] {"2024-09-24T06:15:55.000Z,51.0,"}; + tableResultSetEqualTest( + "select max(time),max(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void maxByTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + String[] retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, max_by(time, s4), max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select device_id, max_by(time, s4), max(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3"}; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "d01,2024-09-24T06:15:50.000Z,null,null,", + "d01,2024-09-24T06:15:55.000Z,null,null,", + }; + + tableResultSetEqualTest( + "select device_id, date_bin(5s, time), max_by(time, s3), max(s3) from table1 where device_id = 'd01' group by date_bin(5s, time), 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,46.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55.0,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),max_by(time, s4), max(s4) from table1 group by 1,2,3,4,date_bin(5s, time) order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id, date_bin(5s, time),max_by(time, s1), max(s1) from table1 group by date_bin(5s, time),1,2,3,4 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,46000,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,40000,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,50000,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,46000,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,46000,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,40000,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,50000,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,46000,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,40000,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,max_by(time, s2), max(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"40,"}; + tableResultSetEqualTest( + "select max_by(s1, s10) from table1 where s1=40", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void firstTest() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "device_id", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_chaoyang_red_B_d10_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_chaoyang_yellow_A_d11_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_chaoyang_yellow_A_d11_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_chaoyang_yellow_A_d11_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_chaoyang_yellow_A_d11_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_haidian_red_A_d13_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_haidian_red_A_d13_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_haidian_red_B_d14_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_haidian_yellow_A_d15_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_haidian_yellow_A_d15_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_haidian_yellow_A_d15_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_haidian_yellow_A_d15_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_haidian_yellow_B_d16_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_huangpu_red_B_d02_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_huangpu_yellow_A_d03_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_huangpu_yellow_A_d03_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_huangpu_yellow_A_d03_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_pudong_red_A_d05_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_pudong_red_A_d05_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_pudong_red_B_d06_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_pudong_yellow_A_d07_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_pudong_yellow_A_d07_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_pudong_yellow_A_d07_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_pudong_yellow_B_d08_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_pudong_yellow_B_d08_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,beijing_haidian_yellow_A_d15_41,beijing_haidian_yellow_A_d15_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,30,35000,30.0,35.0,true,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,36,50000,null,40.0,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,0xcafebabe50,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,36,31000,41.0,36.0,false,shanghai_pudong_yellow_A_d07_41,shanghai_pudong_yellow_A_d07_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,55,40000,30.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, first(time),first(s1),first(s2),first(s3),first(s4),first(s5),first(s6),first(s7),first(s8),first(s9),first(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:35.000Z,35000,", + }; + tableResultSetEqualTest( + "select first_by(time,s2),first(s2) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = + new String[] { + "null,2024-09-24T06:15:30.000Z,", + }; + tableResultSetEqualTest( + "select first_by(s2,time),first(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,null,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,null,null,", + "2024-09-24T06:15:50.000Z,d01,null,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55.0,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, first_by(time, s4), first(s4) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,null,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first_by(time,s3), first(s3) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,beijing_chaoyang_red_A_d09_35,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,beijing_chaoyang_red_B_d10_36,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,beijing_chaoyang_yellow_A_d11_36,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,beijing_chaoyang_yellow_B_d12_30,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,beijing_haidian_red_A_d13_35,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,beijing_haidian_red_B_d14_36,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,beijing_haidian_yellow_A_d15_36,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,beijing_haidian_yellow_B_d16_30,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,shanghai_huangpu_red_A_d01_35,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,shanghai_huangpu_red_B_d02_36,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,shanghai_huangpu_yellow_A_d03_36,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,shanghai_huangpu_yellow_B_d04_30,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,shanghai_pudong_red_A_d05_35,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,shanghai_pudong_red_B_d06_36,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,shanghai_pudong_yellow_A_d07_36,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,shanghai_pudong_yellow_B_d08_30,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,first_by(time,s7),first(s7) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"city", "region", "device_id", "_col3"}; + retArray = + new String[] { + "beijing,chaoyang,d09,null,", + "beijing,chaoyang,d10,true,", + "beijing,chaoyang,d11,null,", + "beijing,chaoyang,d12,true,", + "beijing,haidian,d13,null,", + "beijing,haidian,d14,true,", + "beijing,haidian,d15,null,", + "beijing,haidian,d16,true,", + "shanghai,huangpu,d01,null,", + "shanghai,huangpu,d02,true,", + "shanghai,huangpu,d03,null,", + "shanghai,huangpu,d04,true,", + "shanghai,pudong,d05,null,", + "shanghai,pudong,d06,true,", + "shanghai,pudong,d07,null,", + "shanghai,pudong,d08,true,", + }; + tableResultSetEqualTest( + "select city,region,device_id,first_by(s5,time,time) from table1 group by city,region,device_id order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastTest() { + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "_col0", + "device_id", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "2024-09-24T06:15:40.000Z,d01,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "2024-09-24T06:15:50.000Z,d01,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id, last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_chaoyang_red_B_d10_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_chaoyang_yellow_A_d11_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_chaoyang_yellow_A_d11_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_chaoyang_yellow_A_d11_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_chaoyang_yellow_A_d11_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_haidian_red_A_d13_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_haidian_red_A_d13_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_36,null,2024-09-24T06:15:36.000Z,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,beijing_haidian_red_B_d14_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,beijing_haidian_yellow_A_d15_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,beijing_haidian_yellow_A_d15_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,beijing_haidian_yellow_A_d15_46,null,2024-09-24T06:15:46.000Z,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,beijing_haidian_yellow_A_d15_51,null,null,2024-09-24T06:15:51.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_haidian_yellow_B_d16_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_huangpu_red_B_d02_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_huangpu_yellow_A_d03_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_huangpu_yellow_A_d03_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_huangpu_yellow_A_d03_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_pudong_red_A_d05_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_pudong_red_A_d05_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,null,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_36,null,2024-09-24T06:15:36.000Z,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,null,40.0,null,null,shanghai_pudong_red_B_d06_40,null,2024-09-24T06:15:40.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,2024-09-24T06:15:31.000Z,null,31000,null,null,null,null,null,0xcafebabe31,2024-09-24T06:15:31.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,2024-09-24T06:15:36.000Z,36,null,null,36.0,null,null,shanghai_pudong_yellow_A_d07_36,null,2024-09-24T06:15:36.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41,null,41.0,null,false,shanghai_pudong_yellow_A_d07_41,null,0xcafebabe41,2024-09-24T06:15:41.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,2024-09-24T06:15:46.000Z,null,46000,null,46.0,null,null,shanghai_pudong_yellow_A_d07_46,null,2024-09-24T06:15:46.000Z,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_pudong_yellow_B_d08_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_pudong_yellow_B_d08_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,beijing_chaoyang_red_B_d10_36,beijing_chaoyang_red_B_d10_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,beijing_haidian_red_A_d13_35,beijing_haidian_red_A_d13_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,beijing_haidian_red_B_d14_36,beijing_haidian_red_B_d14_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,beijing_haidian_yellow_A_d15_51,beijing_haidian_yellow_A_d15_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,shanghai_huangpu_red_B_d02_36,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,55,50000,40.0,55.0,false,shanghai_pudong_red_A_d05_35,shanghai_pudong_red_A_d05_40,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,40,50000,null,40.0,true,shanghai_pudong_red_B_d06_36,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,41,46000,51.0,46.0,false,shanghai_pudong_yellow_A_d07_51,shanghai_pudong_yellow_A_d07_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,55,40000,30.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id, last(time),last(s1),last(s2),last(s3),last(s4),last(s5),last(s6),last(s7),last(s8),last(s9),last(s10) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:50.000Z,50000,", + }; + repeatTest( + "select last_by(time,s2),last(s2) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"_col0", "_col1"}; + retArray = + new String[] { + "null,2024-09-24T06:15:55.000Z,", + }; + repeatTest( + "select last_by(s2, time),last(time) from table1 where device_id = 'd01'", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"_col0", "device_id", "_col2", "_col3"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,null,null,", + "2024-09-24T06:15:35.000Z,d01,2024-09-24T06:15:35.000Z,35.0,", + "2024-09-24T06:15:40.000Z,d01,null,null,", + "2024-09-24T06:15:50.000Z,d01,null,null,", + "2024-09-24T06:15:55.000Z,d01,2024-09-24T06:15:55.000Z,55.0,", + }; + repeatTest( + "select date_bin(5s, time), device_id, last_by(time, s4), last(s4) from table1 where device_id = 'd01' group by 1,2", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = + new String[] {"province", "city", "region", "device_id", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:30.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:35.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:45.000Z,null,null,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,null,null,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,35.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40.0,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:30.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:35.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:40.000Z,2024-09-24T06:15:41.000Z,41.0,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:45.000Z,null,null,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:50.000Z,2024-09-24T06:15:51.000Z,51.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30.0,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,null,null,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,null,null,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s, time), first_by(time,s3), first(s3) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,0xcafebabe55,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,0xcafebabe50,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,0xcafebabe41,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,0xcafebabe55,", + }; + repeatTest( + "select province,city,region,device_id,last_by(time,s8),last(s8) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + + expectedHeader = new String[] {"city", "region", "device_id", "_col3"}; + retArray = + new String[] { + "beijing,chaoyang,d09,null,", + "beijing,chaoyang,d10,null,", + "beijing,chaoyang,d11,null,", + "beijing,chaoyang,d12,null,", + "beijing,haidian,d13,null,", + "beijing,haidian,d14,null,", + "beijing,haidian,d15,null,", + "beijing,haidian,d16,null,", + "shanghai,huangpu,d01,null,", + "shanghai,huangpu,d02,null,", + "shanghai,huangpu,d03,null,", + "shanghai,huangpu,d04,null,", + "shanghai,pudong,d05,null,", + "shanghai,pudong,d06,null,", + "shanghai,pudong,d07,null,", + "shanghai,pudong,d08,null,", + }; + repeatTest( + "select city,region,device_id,last_by(s5,time) from table1 group by city,region,device_id order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME, + 2); + } + + @Test + public void extremeTest() { + String[] expectedHeader = new String[] {"device_id", "_col1", "_col2", "_col3"}; + String[] retArray = new String[] {"d01,55,50000,55.0,"}; + + tableResultSetEqualTest( + "select device_id, extreme(s1), extreme(s2), extreme(s4) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {"d01,40.0,"}; + tableResultSetEqualTest( + "select device_id, extreme(s3) from table1 where time >= 2024-09-24T06:15:30.000+00:00 and time <= 2024-09-24T06:15:59.999+00:00 and device_id = 'd01' and s1 >= 40 group by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "device_id", "_col2"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,d01,30.0,", + "2024-09-24T06:15:35.000Z,d01,35.0,", + "2024-09-24T06:15:40.000Z,d01,40.0,", + "2024-09-24T06:15:50.000Z,d01,null,", + "2024-09-24T06:15:55.000Z,d01,null,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time), device_id,extreme(s3) from table1 where device_id = 'd01' group by 1, 2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55.0,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,35.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55.0,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36.0,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,46.0,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55.0,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,35.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55.0,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36.0,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,46.0,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55.0,", + }; + + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,extreme(s4) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0", "province", "city", "region", "device_id", "_col5"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d09,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d09,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d09,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d09,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d10,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d10,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d10,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,chaoyang,d11,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d11,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,chaoyang,d11,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,chaoyang,d12,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,chaoyang,d12,55,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d13,30,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d13,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d13,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d13,55,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d14,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d14,40,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d14,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:35.000Z,beijing,beijing,haidian,d15,36,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d15,41,", + "2024-09-24T06:15:45.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:50.000Z,beijing,beijing,haidian,d15,null,", + "2024-09-24T06:15:30.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:40.000Z,beijing,beijing,haidian,d16,null,", + "2024-09-24T06:15:55.000Z,beijing,beijing,haidian,d16,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d01,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d01,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d01,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d01,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d02,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d02,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d02,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,huangpu,d03,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d03,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,huangpu,d03,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,huangpu,d04,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,huangpu,d04,55,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d05,30,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d05,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d05,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d05,55,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d06,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d06,40,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d06,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:35.000Z,shanghai,shanghai,pudong,d07,36,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d07,41,", + "2024-09-24T06:15:45.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:50.000Z,shanghai,shanghai,pudong,d07,null,", + "2024-09-24T06:15:30.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:40.000Z,shanghai,shanghai,pudong,d08,null,", + "2024-09-24T06:15:55.000Z,shanghai,shanghai,pudong,d08,55,", + }; + tableResultSetEqualTest( + "select date_bin(5s, time),province,city,region,device_id,extreme(s1) from table1 group by 1,2,3,4,5 order by 2,3,4,5,1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,50000,", + "beijing,beijing,chaoyang,d10,50000,", + "beijing,beijing,chaoyang,d11,46000,", + "beijing,beijing,chaoyang,d12,40000,", + "beijing,beijing,haidian,d13,50000,", + "beijing,beijing,haidian,d14,50000,", + "beijing,beijing,haidian,d15,46000,", + "beijing,beijing,haidian,d16,40000,", + "shanghai,shanghai,huangpu,d01,50000,", + "shanghai,shanghai,huangpu,d02,50000,", + "shanghai,shanghai,huangpu,d03,46000,", + "shanghai,shanghai,huangpu,d04,40000,", + "shanghai,shanghai,pudong,d05,50000,", + "shanghai,shanghai,pudong,d06,50000,", + "shanghai,shanghai,pudong,d07,46000,", + "shanghai,shanghai,pudong,d08,40000,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,extreme(s2) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,55.0,", + "beijing,beijing,haidian,55.0,", + "shanghai,shanghai,huangpu,55.0,", + "shanghai,shanghai,pudong,55.0,", + }; + tableResultSetEqualTest( + "select province,city,region,extreme(s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "_col2"}; + retArray = + new String[] { + "beijing,beijing,55.0,", "shanghai,shanghai,55.0,", + }; + tableResultSetEqualTest( + "select province,city,extreme(s4) from table1 group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "_col1"}; + retArray = + new String[] { + "beijing,55.0,", "shanghai,55.0,", + }; + tableResultSetEqualTest( + "select province,extreme(s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = + new String[] { + "51.0,", + }; + tableResultSetEqualTest( + "select extreme(s3) from table1", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void groupByValueTest() { + + String[] expectedHeader = new String[] {"s1", "_col1"}; + String[] retArray = + new String[] { + "30,4,", "36,8,", "40,8,", "41,4,", "55,8,", "null,32,", + }; + tableResultSetEqualTest( + "select s1, count(*) from table1 group by s1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s1", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30,1,", + "beijing,beijing,chaoyang,d09,40,1,", + "beijing,beijing,chaoyang,d09,55,1,", + "beijing,beijing,chaoyang,d09,null,2,", + "beijing,beijing,chaoyang,d10,36,1,", + "beijing,beijing,chaoyang,d10,40,1,", + "beijing,beijing,chaoyang,d10,null,1,", + "beijing,beijing,chaoyang,d11,36,1,", + "beijing,beijing,chaoyang,d11,41,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,30,1,", + "beijing,beijing,haidian,d13,40,1,", + "beijing,beijing,haidian,d13,55,1,", + "beijing,beijing,haidian,d13,null,2,", + "beijing,beijing,haidian,d14,36,1,", + "beijing,beijing,haidian,d14,40,1,", + "beijing,beijing,haidian,d14,null,1,", + "beijing,beijing,haidian,d15,36,1,", + "beijing,beijing,haidian,d15,41,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,30,1,", + "shanghai,shanghai,huangpu,d01,40,1,", + "shanghai,shanghai,huangpu,d01,55,1,", + "shanghai,shanghai,huangpu,d01,null,2,", + "shanghai,shanghai,huangpu,d02,36,1,", + "shanghai,shanghai,huangpu,d02,40,1,", + "shanghai,shanghai,huangpu,d02,null,1,", + "shanghai,shanghai,huangpu,d03,36,1,", + "shanghai,shanghai,huangpu,d03,41,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,30,1,", + "shanghai,shanghai,pudong,d05,40,1,", + "shanghai,shanghai,pudong,d05,55,1,", + "shanghai,shanghai,pudong,d05,null,2,", + "shanghai,shanghai,pudong,d06,36,1,", + "shanghai,shanghai,pudong,d06,40,1,", + "shanghai,shanghai,pudong,d06,null,1,", + "shanghai,shanghai,pudong,d07,36,1,", + "shanghai,shanghai,pudong,d07,41,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s1,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s2", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,35000,1,", + "beijing,beijing,chaoyang,d09,50000,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,50000,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,31000,1,", + "beijing,beijing,chaoyang,d11,46000,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,40000,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,35000,1,", + "beijing,beijing,haidian,d13,50000,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,50000,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,31000,1,", + "beijing,beijing,haidian,d15,46000,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,40000,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,35000,1,", + "shanghai,shanghai,huangpu,d01,50000,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,50000,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,31000,1,", + "shanghai,shanghai,huangpu,d03,46000,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,40000,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,35000,1,", + "shanghai,shanghai,pudong,d05,50000,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,50000,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,31000,1,", + "shanghai,shanghai,pudong,d07,46000,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,40000,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s2,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s3", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30.0,1,", + "beijing,beijing,chaoyang,d09,35.0,1,", + "beijing,beijing,chaoyang,d09,40.0,1,", + "beijing,beijing,chaoyang,d09,null,2,", + "beijing,beijing,chaoyang,d10,null,3,", + "beijing,beijing,chaoyang,d11,41.0,1,", + "beijing,beijing,chaoyang,d11,51.0,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,30.0,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,30.0,1,", + "beijing,beijing,haidian,d13,35.0,1,", + "beijing,beijing,haidian,d13,40.0,1,", + "beijing,beijing,haidian,d13,null,2,", + "beijing,beijing,haidian,d14,null,3,", + "beijing,beijing,haidian,d15,41.0,1,", + "beijing,beijing,haidian,d15,51.0,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,30.0,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,30.0,1,", + "shanghai,shanghai,huangpu,d01,35.0,1,", + "shanghai,shanghai,huangpu,d01,40.0,1,", + "shanghai,shanghai,huangpu,d01,null,2,", + "shanghai,shanghai,huangpu,d02,null,3,", + "shanghai,shanghai,huangpu,d03,41.0,1,", + "shanghai,shanghai,huangpu,d03,51.0,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,30.0,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,30.0,1,", + "shanghai,shanghai,pudong,d05,35.0,1,", + "shanghai,shanghai,pudong,d05,40.0,1,", + "shanghai,shanghai,pudong,d05,null,2,", + "shanghai,shanghai,pudong,d06,null,3,", + "shanghai,shanghai,pudong,d07,41.0,1,", + "shanghai,shanghai,pudong,d07,51.0,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,30.0,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s3,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,35.0,1,", + "beijing,beijing,chaoyang,d09,55.0,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,40.0,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,36.0,1,", + "beijing,beijing,chaoyang,d11,46.0,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,55.0,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,35.0,1,", + "beijing,beijing,haidian,d13,55.0,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,40.0,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,36.0,1,", + "beijing,beijing,haidian,d15,46.0,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,55.0,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,35.0,1,", + "shanghai,shanghai,huangpu,d01,55.0,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,40.0,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,36.0,1,", + "shanghai,shanghai,huangpu,d03,46.0,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,55.0,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,35.0,1,", + "shanghai,shanghai,pudong,d05,55.0,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,40.0,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,36.0,1,", + "shanghai,shanghai,pudong,d07,46.0,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,55.0,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s4,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s5", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,false,1,", + "beijing,beijing,chaoyang,d09,true,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,true,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,false,1,", + "beijing,beijing,chaoyang,d11,null,4,", + "beijing,beijing,chaoyang,d12,true,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,false,1,", + "beijing,beijing,haidian,d13,true,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,true,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,false,1,", + "beijing,beijing,haidian,d15,null,4,", + "beijing,beijing,haidian,d16,true,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,false,1,", + "shanghai,shanghai,huangpu,d01,true,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,true,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,false,1,", + "shanghai,shanghai,huangpu,d03,null,4,", + "shanghai,shanghai,huangpu,d04,true,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,false,1,", + "shanghai,shanghai,pudong,d05,true,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,true,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,false,1,", + "shanghai,shanghai,pudong,d07,null,4,", + "shanghai,shanghai,pudong,d08,true,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s5,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s6", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_30,1,", + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_35,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_36,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_41,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_51,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,beijing_chaoyang_yellow_B_d12_55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_30,1,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_35,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_36,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_41,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_51,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,beijing_haidian_yellow_B_d16_55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_30,1,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_35,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_36,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_41,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_51,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,shanghai_huangpu_yellow_B_d04_55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_30,1,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_35,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_36,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_41,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_51,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,shanghai_pudong_yellow_B_d08_55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s6,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s7", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_35,1,", + "beijing,beijing,chaoyang,d09,beijing_chaoyang_red_A_d09_40,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_36,1,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_40,1,", + "beijing,beijing,chaoyang,d10,beijing_chaoyang_red_B_d10_50,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_36,1,", + "beijing,beijing,chaoyang,d11,beijing_chaoyang_yellow_A_d11_46,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,beijing_chaoyang_yellow_B_d12_30,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_35,1,", + "beijing,beijing,haidian,d13,beijing_haidian_red_A_d13_40,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_36,1,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_40,1,", + "beijing,beijing,haidian,d14,beijing_haidian_red_B_d14_50,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_36,1,", + "beijing,beijing,haidian,d15,beijing_haidian_yellow_A_d15_46,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,beijing_haidian_yellow_B_d16_30,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_35,1,", + "shanghai,shanghai,huangpu,d01,shanghai_huangpu_red_A_d01_40,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_36,1,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_40,1,", + "shanghai,shanghai,huangpu,d02,shanghai_huangpu_red_B_d02_50,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_36,1,", + "shanghai,shanghai,huangpu,d03,shanghai_huangpu_yellow_A_d03_46,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,shanghai_huangpu_yellow_B_d04_30,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_35,1,", + "shanghai,shanghai,pudong,d05,shanghai_pudong_red_A_d05_40,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_36,1,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_40,1,", + "shanghai,shanghai,pudong,d06,shanghai_pudong_red_B_d06_50,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_36,1,", + "shanghai,shanghai,pudong,d07,shanghai_pudong_yellow_A_d07_46,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,shanghai_pudong_yellow_B_d08_30,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s7,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select province,city,region,device_id,s7,count(1) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s8", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,0xcafebabe30,1,", + "beijing,beijing,chaoyang,d09,0xcafebabe55,1,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,0xcafebabe50,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,0xcafebabe31,1,", + "beijing,beijing,chaoyang,d11,0xcafebabe41,1,", + "beijing,beijing,chaoyang,d11,null,3,", + "beijing,beijing,chaoyang,d12,0xcafebabe55,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,0xcafebabe30,1,", + "beijing,beijing,haidian,d13,0xcafebabe55,1,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,0xcafebabe50,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,0xcafebabe31,1,", + "beijing,beijing,haidian,d15,0xcafebabe41,1,", + "beijing,beijing,haidian,d15,null,3,", + "beijing,beijing,haidian,d16,0xcafebabe55,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,0xcafebabe30,1,", + "shanghai,shanghai,huangpu,d01,0xcafebabe55,1,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,0xcafebabe50,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,0xcafebabe31,1,", + "shanghai,shanghai,huangpu,d03,0xcafebabe41,1,", + "shanghai,shanghai,huangpu,d03,null,3,", + "shanghai,shanghai,huangpu,d04,0xcafebabe55,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,0xcafebabe30,1,", + "shanghai,shanghai,pudong,d05,0xcafebabe55,1,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,0xcafebabe50,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,0xcafebabe31,1,", + "shanghai,shanghai,pudong,d07,0xcafebabe41,1,", + "shanghai,shanghai,pudong,d07,null,3,", + "shanghai,shanghai,pudong,d08,0xcafebabe55,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s8,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s9", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d10,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:31.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:41.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:46.000Z,1,", + "beijing,beijing,chaoyang,d11,2024-09-24T06:15:51.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:35.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,haidian,d13,2024-09-24T06:15:55.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d14,2024-09-24T06:15:50.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:31.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:36.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:41.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:46.000Z,1,", + "beijing,beijing,haidian,d15,2024-09-24T06:15:51.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:30.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:40.000Z,1,", + "beijing,beijing,haidian,d16,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d02,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:31.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:41.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:46.000Z,1,", + "shanghai,shanghai,huangpu,d03,2024-09-24T06:15:51.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:35.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,pudong,d05,2024-09-24T06:15:55.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d06,2024-09-24T06:15:50.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:31.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:36.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:41.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:46.000Z,1,", + "shanghai,shanghai,pudong,d07,2024-09-24T06:15:51.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:30.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:40.000Z,1,", + "shanghai,shanghai,pudong,d08,2024-09-24T06:15:55.000Z,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s9,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "s10", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24,2,", + "beijing,beijing,chaoyang,d09,null,3,", + "beijing,beijing,chaoyang,d10,2024-09-24,1,", + "beijing,beijing,chaoyang,d10,null,2,", + "beijing,beijing,chaoyang,d11,2024-09-24,1,", + "beijing,beijing,chaoyang,d11,null,4,", + "beijing,beijing,chaoyang,d12,2024-09-24,1,", + "beijing,beijing,chaoyang,d12,null,2,", + "beijing,beijing,haidian,d13,2024-09-24,2,", + "beijing,beijing,haidian,d13,null,3,", + "beijing,beijing,haidian,d14,2024-09-24,1,", + "beijing,beijing,haidian,d14,null,2,", + "beijing,beijing,haidian,d15,2024-09-24,1,", + "beijing,beijing,haidian,d15,null,4,", + "beijing,beijing,haidian,d16,2024-09-24,1,", + "beijing,beijing,haidian,d16,null,2,", + "shanghai,shanghai,huangpu,d01,2024-09-24,2,", + "shanghai,shanghai,huangpu,d01,null,3,", + "shanghai,shanghai,huangpu,d02,2024-09-24,1,", + "shanghai,shanghai,huangpu,d02,null,2,", + "shanghai,shanghai,huangpu,d03,2024-09-24,1,", + "shanghai,shanghai,huangpu,d03,null,4,", + "shanghai,shanghai,huangpu,d04,2024-09-24,1,", + "shanghai,shanghai,huangpu,d04,null,2,", + "shanghai,shanghai,pudong,d05,2024-09-24,2,", + "shanghai,shanghai,pudong,d05,null,3,", + "shanghai,shanghai,pudong,d06,2024-09-24,1,", + "shanghai,shanghai,pudong,d06,null,2,", + "shanghai,shanghai,pudong,d07,2024-09-24,1,", + "shanghai,shanghai,pudong,d07,null,4,", + "shanghai,shanghai,pudong,d08,2024-09-24,1,", + "shanghai,shanghai,pudong,d08,null,2,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,s10,count(*) from table1 group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastQueryTest() { + + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id='d01'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d09,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d12,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id,date_bin(5s,time),last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by province,city,region,device_id,date_bin(5s,time) order by device_id,date_bin(5s,time)", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "device_id", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13", + "_col14", + "_col15" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,beijing_chaoyang_red_A_d09_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,beijing_chaoyang_red_A_d09_35,beijing_chaoyang_red_A_d09_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,beijing_chaoyang_red_A_d09_40,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d09,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,beijing_chaoyang_yellow_B_d12_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "beijing,beijing,chaoyang,d12,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_chaoyang_yellow_B_d12_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,30,null,30.0,null,null,shanghai_huangpu_red_A_d01_30,null,0xcafebabe30,2024-09-24T06:15:30.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:35.000Z,2024-09-24T06:15:35.000Z,null,35000,35.0,35.0,null,shanghai_huangpu_red_A_d01_35,shanghai_huangpu_red_A_d01_35,null,2024-09-24T06:15:35.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,40,null,40.0,null,true,null,shanghai_huangpu_red_A_d01_40,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:50.000Z,2024-09-24T06:15:50.000Z,null,50000,null,null,false,null,null,null,2024-09-24T06:15:50.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:30.000Z,2024-09-24T06:15:30.000Z,null,null,30.0,null,true,null,shanghai_huangpu_yellow_B_d04_30,null,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:40.000Z,2024-09-24T06:15:40.000Z,null,40000,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "shanghai,shanghai,huangpu,d04,2024-09-24T06:15:55.000Z,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,device_id,date_bin(5s,time),last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id in ('d01', 'd04', 'd09', 'd12') group by 1,2,3,4,5 order by 1,2,3,4,5", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "beijing,beijing,haidian,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,pudong,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,region,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd08', 'd12', 'd13') group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,shanghai,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,city,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd05', 'd08', 'd09', 'd12', 'd13') group by 1,2 order by 1,2", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "beijing,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,beijing_haidian_yellow_B_d16_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "shanghai,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + }; + + tableResultSetEqualTest( + "select province,last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where device_id NOT in ('d01', 'd05', 'd08', 'd09', 'd12', 'd13') group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", + "_col1", + "_col2", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11" + }; + retArray = + new String[] { + "d01,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d02,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_huangpu_red_B_d02_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "d03,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_huangpu_yellow_A_d03_51,null,null,2024-09-24T06:15:51.000Z,null,", + "d04,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,shanghai_huangpu_yellow_B_d04_55,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d05,2024-09-24T06:15:55.000Z,55,null,null,55.0,null,null,null,0xcafebabe55,2024-09-24T06:15:55.000Z,null,", + "d06,2024-09-24T06:15:50.000Z,null,50000,null,null,null,null,shanghai_pudong_red_B_d06_50,0xcafebabe50,2024-09-24T06:15:50.000Z,null,", + "d07,2024-09-24T06:15:51.000Z,null,null,51.0,null,null,shanghai_pudong_yellow_A_d07_51,null,null,2024-09-24T06:15:51.000Z,null,", + }; + + tableResultSetEqualTest( + "select device_id, last(time),last_by(s1,time),last_by(s2,time),last_by(s3,time),last_by(s4,time),last_by(s5,time),last_by(s6,time),last_by(s7,time),last_by(s8,time),last_by(s9,time),last_by(s10,time) from table1 where city = 'shanghai' group by province,city,region,device_id order by device_id limit 7", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void specialCasesTest() { + String[] expectedHeader = new String[] {"device_id"}; + String[] retArray = + new String[] { + "d01,", "d02,", "d03,", "d04,", "d05,", "d06,", "d07,", "d08,", "d09,", "d10,", "d11,", + "d12,", "d13,", "d14,", "d15,", "d16,", + }; + tableResultSetEqualTest( + "SELECT device_id FROM table1 GROUP BY device_id order by device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time"}; + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,", + "2024-09-24T06:15:31.000Z,", + "2024-09-24T06:15:35.000Z,", + "2024-09-24T06:15:36.000Z,", + "2024-09-24T06:15:40.000Z,", + "2024-09-24T06:15:41.000Z,", + "2024-09-24T06:15:46.000Z,", + "2024-09-24T06:15:50.000Z,", + "2024-09-24T06:15:51.000Z,", + "2024-09-24T06:15:55.000Z," + }; + tableResultSetEqualTest( + "select time from table1 group by time order by time", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void modeTest() { + // AggTableScan + Agg mixed test + String[] expectedHeader = buildHeaders(10); + String[] retArray = + new String[] { + "null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + }; + tableResultSetEqualTest( + "select mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6),mode(s7),mode(s8),mode(s9),mode(s10) from table1 group by city", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void approxCountDistinctTest() { + String[] expectedHeader = buildHeaders(15); + String[] retArray = new String[] {"10,2,2,4,16,5,5,5,5,2,24,32,5,10,1,"}; + tableResultSetEqualTest( + "select approx_count_distinct(time), approx_count_distinct(province), approx_count_distinct(city), approx_count_distinct(region), approx_count_distinct(device_id), approx_count_distinct(s1), approx_count_distinct(s2), approx_count_distinct(s3), approx_count_distinct(s4), approx_count_distinct(s5), approx_count_distinct(s6), approx_count_distinct(s7), approx_count_distinct(s8), approx_count_distinct(s9), approx_count_distinct(s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select approx_count_distinct(time, 0.02), approx_count_distinct(province, 0.02), approx_count_distinct(city, 0.02), approx_count_distinct(region, 0.02), approx_count_distinct(device_id, 0.02), approx_count_distinct(s1, 0.02), approx_count_distinct(s2, 0.02), approx_count_distinct(s3, 0.02), approx_count_distinct(s4, 0.02), approx_count_distinct(s5, 0.02), approx_count_distinct(s6, 0.02), approx_count_distinct(s7, 0.02), approx_count_distinct(s8, 0.02), approx_count_distinct(s9, 0.02), approx_count_distinct(s10, 0.02) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + "2024-09-24T06:15:30.000Z,beijing,2,2,", + "2024-09-24T06:15:31.000Z,beijing,0,0,", + "2024-09-24T06:15:35.000Z,beijing,2,2,", + "2024-09-24T06:15:36.000Z,beijing,2,4,", + "2024-09-24T06:15:40.000Z,beijing,0,4,", + "2024-09-24T06:15:41.000Z,beijing,2,0,", + "2024-09-24T06:15:46.000Z,beijing,0,2,", + "2024-09-24T06:15:50.000Z,beijing,0,2,", + "2024-09-24T06:15:51.000Z,beijing,2,0,", + "2024-09-24T06:15:55.000Z,beijing,2,0,", + "2024-09-24T06:15:30.000Z,shanghai,2,2,", + "2024-09-24T06:15:31.000Z,shanghai,0,0,", + "2024-09-24T06:15:35.000Z,shanghai,2,2,", + "2024-09-24T06:15:36.000Z,shanghai,2,4,", + "2024-09-24T06:15:40.000Z,shanghai,0,4,", + "2024-09-24T06:15:41.000Z,shanghai,2,0,", + "2024-09-24T06:15:46.000Z,shanghai,0,2,", + "2024-09-24T06:15:50.000Z,shanghai,0,2,", + "2024-09-24T06:15:51.000Z,shanghai,2,0,", + "2024-09-24T06:15:55.000Z,shanghai,2,0,", + }; + + tableResultSetEqualTest( + "select time,province,approx_count_distinct(s6),approx_count_distinct(s7) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2", "_col3"}, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select time,province,approx_count_distinct(s6,0.02),approx_count_distinct(s7,0.02) from table1 group by 1,2 order by 2,1", + new String[] {"time", "province", "_col2", "_col3"}, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest() { + tableAssertTestFail( + "select avg() from table1", + "701: Aggregate functions [avg] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select sum() from table1", + "701: Aggregate functions [sum] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select extreme() from table1", + "701: Aggregate functions [extreme] should only have one argument", + DATABASE_NAME); + tableAssertTestFail( + "select first() from table1", + "701: Aggregate functions [first] should only have one or two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select first_by() from table1", + "701: Aggregate functions [first_by] should only have two or three arguments", + DATABASE_NAME); + tableAssertTestFail( + "select last() from table1", + "701: Aggregate functions [last] should only have one or two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select last_by() from table1", + "701: Aggregate functions [last_by] should only have two or three arguments", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct() from table1", + "701: Aggregate functions [approx_count_distinct] should only have two arguments", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 0.3) from table1", + "750: Max Standard Error must be in [0.0040625, 0.26]: 0.3", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 0.3) from table1", + "750: Max Standard Error must be in [0.0040625, 0.26]: 0.3", + DATABASE_NAME); + tableAssertTestFail( + "select approx_count_distinct(province, 'test') from table1", + "701: Second argument of Aggregate functions [approx_count_distinct] should be numberic type and do not use expression", + DATABASE_NAME); + } + + // ================================================================== + // ===================== Select Distinct Test ======================= + // ================================================================== + + // Select distinct is a special kind of aggregate query in actual, so we put ITs here to reuse the + // test data. + + @Test + public void simpleTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = new String[] {"30,", "36,", "40,", "41,", "55,", "null,"}; + tableResultSetEqualTest( + "select distinct s1 from table1 order by s1", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"region", "s1"}; + retArray = + new String[] { + "chaoyang,30,", + "chaoyang,36,", + "chaoyang,40,", + "chaoyang,41,", + "chaoyang,55,", + "chaoyang,null,", + "haidian,30,", + "haidian,36,", + "haidian,40,", + "haidian,41,", + "haidian,55,", + "haidian,null,", + "huangpu,30,", + "huangpu,36,", + "huangpu,40,", + "huangpu,41,", + "huangpu,55,", + "huangpu,null,", + "pudong,30,", + "pudong,36,", + "pudong,40,", + "pudong,41,", + "pudong,55,", + "pudong,null," + }; + tableResultSetEqualTest( + "select distinct region, s1 from table1 order by region, s1", + expectedHeader, + retArray, + DATABASE_NAME); + + // show all devices + expectedHeader = new String[] {"province", "city", "region", "device_id"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,", + "beijing,beijing,chaoyang,d10,", + "beijing,beijing,chaoyang,d11,", + "beijing,beijing,chaoyang,d12,", + "beijing,beijing,haidian,d13,", + "beijing,beijing,haidian,d14,", + "beijing,beijing,haidian,d15,", + "beijing,beijing,haidian,d16,", + "shanghai,shanghai,huangpu,d01,", + "shanghai,shanghai,huangpu,d02,", + "shanghai,shanghai,huangpu,d03,", + "shanghai,shanghai,huangpu,d04,", + "shanghai,shanghai,pudong,d05,", + "shanghai,shanghai,pudong,d06,", + "shanghai,shanghai,pudong,d07,", + "shanghai,shanghai,pudong,d08,", + }; + tableResultSetEqualTest( + "select distinct province,city,region,device_id from table1 order by province,city,region,device_id", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,"}; + tableResultSetEqualTest( + "select distinct s1 < 0 from table1 where s1 is not null", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void withGroupByTest() { + String[] expectedHeader = new String[] {"s1"}; + String[] retArray = new String[] {"30,", "36,", "40,", "41,", "55,", "null,"}; + tableResultSetEqualTest( + "select distinct s1 from table1 group by s1 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct s1 from table1 group by s1,s2 order by s1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"30.0,", "36.0,", "40.0,", "41.0,", "55.0,", "null,"}; + tableResultSetEqualTest( + "select distinct avg(s1) from table1 group by s1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "select distinct avg(s1) from table1 group by s1,s2 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = new String[] {"4,", "8,", "32,"}; + tableResultSetEqualTest( + "select distinct count(*) from table1 group by s1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = new String[] {"4,", "8,"}; + tableResultSetEqualTest( + "select distinct count(*) from table1 group by s1, s2 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest1() { + tableAssertTestFail( + "select distinct s1 from table1 order by s2", + "701: For SELECT DISTINCT, ORDER BY expressions must appear in select list", + DATABASE_NAME); + } + + // ================================================================== + // ================== Agg-Function Distinct Test ==================== + // ================================================================== + @Test + public void countDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "2,2,4,16,5,5,5,5,2,24,32,5,10,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select count(distinct province), count(distinct city), count(distinct region), count(distinct device_id), count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,5,5,5,2,6,8,5,10,1,", + "beijing,beijing,haidian,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,huangpu,5,5,5,5,2,6,8,5,10,1,", + "shanghai,shanghai,pudong,5,5,5,5,2,6,8,5,10,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,5,5,5,5,2,6,8,5,10,1,", + "haidian,5,5,5,5,2,6,8,5,10,1,", + "huangpu,5,5,5,5,2,6,8,5,10,1,", + "pudong,5,5,5,5,2,6,8,5,10,1," + }; + tableResultSetEqualTest( + "select region, count(distinct s1), count(distinct s2), count(distinct s3), count(distinct s4), count(distinct s5), count(distinct s6), count(distinct s7), count(distinct s8), count(distinct s9), count(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void countIfDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "1,1,1,1,0,1,1,1,1,1,1,1,1,1,", + }; + // global Aggregation + tableResultSetEqualTest( + "select count_if(distinct province = 'shanghai'), count_if(distinct city = 'shanghai'), count_if(distinct region= 'huangpu'), count_if(distinct device_id = 'd03'), count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,0,1,1,1,1,1,1,1,1,1,", + "beijing,beijing,haidian,0,1,1,1,1,1,1,1,1,1,", + "shanghai,shanghai,huangpu,0,1,1,1,1,1,1,1,1,1,", + "shanghai,shanghai,pudong,0,1,1,1,1,1,1,1,1,1," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,0,1,1,1,1,1,1,1,1,1,", + "haidian,0,1,1,1,1,1,1,1,1,1,", + "huangpu,0,1,1,1,1,1,1,1,1,1,", + "pudong,0,1,1,1,1,1,1,1,1,1," + }; + tableResultSetEqualTest( + "select region, count_if(distinct s1 < 0), count_if(distinct s2 is not null), count_if(distinct s3 is not null), count_if(distinct s4 is not null), count_if(distinct s5 is not null), count_if(distinct s6 is not null), count_if(distinct s7 is not null), count_if(distinct s8 is not null), count_if(distinct s9 is not null), count_if(distinct s10 is not null) " + + "from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void avgDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "40.4,40400.0,39.4,42.4,", + }; + // global Aggregation + tableResultSetEqualTest( + "select avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,40.4,40400.0,39.4,42.4,", + "beijing,beijing,haidian,40.4,40400.0,39.4,42.4,", + "shanghai,shanghai,huangpu,40.4,40400.0,39.4,42.4,", + "shanghai,shanghai,pudong,40.4,40400.0,39.4,42.4," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,40.4,40400.0,39.4,42.4,", + "haidian,40.4,40400.0,39.4,42.4,", + "huangpu,40.4,40400.0,39.4,42.4,", + "pudong,40.4,40400.0,39.4,42.4," + }; + tableResultSetEqualTest( + "select region, avg(distinct s1), avg(distinct s2), avg(distinct s3), avg(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void sumDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "202.0,202000.0,197.0,212.0,", + }; + // global Aggregation + tableResultSetEqualTest( + "select sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,202.0,202000.0,197.0,212.0,", + "beijing,beijing,haidian,202.0,202000.0,197.0,212.0,", + "shanghai,shanghai,huangpu,202.0,202000.0,197.0,212.0,", + "shanghai,shanghai,pudong,202.0,202000.0,197.0,212.0," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,202.0,202000.0,197.0,212.0,", + "haidian,202.0,202000.0,197.0,212.0,", + "huangpu,202.0,202000.0,197.0,212.0,", + "pudong,202.0,202000.0,197.0,212.0," + }; + tableResultSetEqualTest( + "select region, sum(distinct s1), sum(distinct s2), sum(distinct s3), sum(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "beijing,beijing,chaoyang,d01,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select min(distinct province), min(distinct city), min(distinct region), min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "beijing,beijing,haidian,d13,30,31000,30.0,35.0,false,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d01,30,31000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d05,30,31000,30.0,35.0,false,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + retArray = + new String[] { + "chaoyang,d09,30,31000,30.0,35.0,false,beijing_chaoyang_red_A_d09_30,beijing_chaoyang_red_A_d09_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "haidian,d13,30,31000,30.0,35.0,false,beijing_haidian_red_A_d13_30,beijing_haidian_red_A_d13_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "huangpu,d01,30,31000,30.0,35.0,false,shanghai_huangpu_red_A_d01_30,shanghai_huangpu_red_A_d01_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24,", + "pudong,d05,30,31000,30.0,35.0,false,shanghai_pudong_red_A_d05_30,shanghai_pudong_red_A_d05_35,0xcafebabe30,2024-09-24T06:15:30.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, min(distinct device_id), min(distinct s1), min(distinct s2), min(distinct s3), min(distinct s4), min(distinct s5), min(distinct s6), min(distinct s7), min(distinct s8), min(distinct s9), min(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void minByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + retArray = + new String[] { + "d03,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "d11,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z," + }; + + // test GroupByDistinctAccumulator + tableResultSetEqualTest( + "select device_id, min_by(distinct time, s1), min_by(distinct time, s2), min_by(distinct time, s3), min_by(distinct time, s4), min_by(distinct time, s5), min_by(distinct time, s6), min_by(distinct time, s9), min_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11", "_col12", "_col13" + }; + String[] retArray = + new String[] { + "shanghai,shanghai,pudong,d16,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select max(distinct province), max(distinct city), max(distinct region), max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12", + "_col13" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,d12,55,50000,51.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "beijing,beijing,haidian,d16,55,50000,51.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,d04,55,50000,51.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "shanghai,shanghai,pudong,d08,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + retArray = + new String[] { + "chaoyang,d12,55,50000,51.0,55.0,true,beijing_chaoyang_yellow_B_d12_55,beijing_chaoyang_yellow_B_d12_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "haidian,d16,55,50000,51.0,55.0,true,beijing_haidian_yellow_B_d16_55,beijing_haidian_yellow_B_d16_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "huangpu,d04,55,50000,51.0,55.0,true,shanghai_huangpu_yellow_B_d04_55,shanghai_huangpu_yellow_B_d04_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24,", + "pudong,d08,55,50000,51.0,55.0,true,shanghai_pudong_yellow_B_d08_55,shanghai_pudong_yellow_B_d08_30,0xcafebabe55,2024-09-24T06:15:55.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region,max(distinct device_id), max(distinct s1), max(distinct s2), max(distinct s3), max(distinct s4), max(distinct s5), max(distinct s6), max(distinct s7), max(distinct s8), max(distinct s9), max(distinct s10) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void maxByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7"}; + String[] retArray = + new String[] { + "2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "device_id", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8" + }; + retArray = + new String[] { + "d03,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "d11,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z," + }; + + // test GroupByDistinctAccumulator + tableResultSetEqualTest( + "select device_id, max_by(distinct time, s1), max_by(distinct time, s2), max_by(distinct time, s3), max_by(distinct time, s4), max_by(distinct time, s5), max_by(distinct time, s6), max_by(distinct time, s9), max_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,36,31000,41.0,36.0,false,beijing_chaoyang_yellow_A_d11_41,beijing_chaoyang_yellow_A_d11_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24,", + "huangpu,36,31000,41.0,36.0,false,shanghai_huangpu_yellow_A_d03_41,shanghai_huangpu_yellow_A_d03_36,0xcafebabe31,2024-09-24T06:15:31.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, first(distinct s1), first(distinct s2), first(distinct s3), first(distinct s4), first(distinct s5), first(distinct s6), first(distinct s7), first(distinct s8), first(distinct s9), first(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void firstByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + // global Aggregation + tableResultSetEqualTest( + "select first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + "huangpu,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:36.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:31.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select region, first_by(distinct time, s1), first_by(distinct time, s2), first_by(distinct time, s3), first_by(distinct time, s4), first_by(distinct time, s5), first_by(distinct time, s6), first_by(distinct time, s7), first_by(distinct time, s8), first_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + }; + // global Aggregation + tableResultSetEqualTest( + "select last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "shanghai,shanghai,huangpu,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,41,46000,51.0,46.0,false,beijing_chaoyang_yellow_A_d11_51,beijing_chaoyang_yellow_A_d11_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24,", + "huangpu,41,46000,51.0,46.0,false,shanghai_huangpu_yellow_A_d03_51,shanghai_huangpu_yellow_A_d03_46,0xcafebabe41,2024-09-24T06:15:51.000Z,2024-09-24," + }; + tableResultSetEqualTest( + "select region, last(distinct s1), last(distinct s2), last(distinct s3), last(distinct s4), last(distinct s5), last(distinct s6), last(distinct s7), last(distinct s8), last(distinct s9), last(distinct s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void lastByDistinctTest() { + // test MarkDistinct + String[] expectedHeader = + new String[] { + "_col0", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9" + }; + String[] retArray = + new String[] { + "2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + // global Aggregation + tableResultSetEqualTest( + "select last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), last_by(distinct time, s10) " + + "from table1 where device_id='d11'", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] { + "province", + "city", + "region", + "_col3", + "_col4", + "_col5", + "_col6", + "_col7", + "_col8", + "_col9", + "_col10", + "_col11", + "_col12" + }; + retArray = + new String[] { + "beijing,beijing,chaoyang,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "shanghai,shanghai,huangpu,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = + new String[] { + "region", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10" + }; + retArray = + new String[] { + "chaoyang,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + "huangpu,2024-09-24T06:15:41.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:46.000Z,2024-09-24T06:15:41.000Z,2024-09-24T06:15:51.000Z,2024-09-24T06:15:36.000Z,", + }; + tableResultSetEqualTest( + "select region, last_by(distinct time, s1), last_by(distinct time, s2), last_by(distinct time, s3), last_by(distinct time, s4), last_by(distinct time, s5), last_by(distinct time, s6), last_by(distinct time, s7), last_by(distinct time, s8), last_by(distinct time, s9), first_by(distinct time, s10) " + + "from table1 where device_id='d11' or device_id='d03' group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extremeDistinctTest() { + // test MarkDistinct + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "55,50000,51.0,55.0,", + }; + // global Aggregation + tableResultSetEqualTest( + "select extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,55,50000,51.0,55.0,", + "beijing,beijing,haidian,55,50000,51.0,55.0,", + "shanghai,shanghai,huangpu,55,50000,51.0,55.0,", + "shanghai,shanghai,pudong,55,50000,51.0,55.0," + }; + // group by Aggregation + tableResultSetEqualTest( + "select province,city,region, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,55,50000,51.0,55.0,", + "haidian,55,50000,51.0,55.0,", + "huangpu,55,50000,51.0,55.0,", + "pudong,55,50000,51.0,55.0," + }; + tableResultSetEqualTest( + "select region, extreme(distinct s1), extreme(distinct s2), extreme(distinct s3), extreme(distinct s4) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void varianceDistinctTest() { + // The addInput logic of all variance functions are the same, so test any one function is ok. + String[] expectedHeader = new String[] {"_col0", "_col1", "_col2", "_col3"}; + String[] retArray = + new String[] { + "68.2,4.824E7,49.0,54.6,", + }; + tableResultSetEqualTest( + "select round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = + new String[] {"province", "city", "region", "_col3", "_col4", "_col5", "_col6"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,68.2,4.824E7,49.0,54.6,", + "beijing,beijing,haidian,68.2,4.824E7,49.0,54.6,", + "shanghai,shanghai,huangpu,68.2,4.824E7,49.0,54.6,", + "shanghai,shanghai,pudong,68.2,4.824E7,49.0,54.6," + }; + tableResultSetEqualTest( + "select province,city,region, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + + // test GroupByDistinctAccumulator + expectedHeader = new String[] {"region", "_col1", "_col2", "_col3", "_col4"}; + retArray = + new String[] { + "chaoyang,68.2,4.824E7,49.0,54.6,", + "haidian,68.2,4.824E7,49.0,54.6,", + "huangpu,68.2,4.824E7,49.0,54.6,", + "pudong,68.2,4.824E7,49.0,54.6," + }; + tableResultSetEqualTest( + "select region, round(VAR_POP(distinct s1),1), round(VAR_POP(distinct s2),1), round(VAR_POP(distinct s3),1), round(VAR_POP(distinct s4),1) from table1 group by 1 order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void mixedTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "32,5,", + }; + + tableResultSetEqualTest( + "select count(s1), count(distinct s1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "device_id", "_col4", "_col5"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,d09,3,3,", + "beijing,beijing,chaoyang,d10,2,2,", + "beijing,beijing,chaoyang,d11,2,2,", + "beijing,beijing,chaoyang,d12,1,1,", + "beijing,beijing,haidian,d13,3,3,", + "beijing,beijing,haidian,d14,2,2,", + "beijing,beijing,haidian,d15,2,2,", + "beijing,beijing,haidian,d16,1,1,", + "shanghai,shanghai,huangpu,d01,3,3,", + "shanghai,shanghai,huangpu,d02,2,2,", + "shanghai,shanghai,huangpu,d03,2,2,", + "shanghai,shanghai,huangpu,d04,1,1,", + "shanghai,shanghai,pudong,d05,3,3,", + "shanghai,shanghai,pudong,d06,2,2,", + "shanghai,shanghai,pudong,d07,2,2,", + "shanghai,shanghai,pudong,d08,1,1,", + }; + tableResultSetEqualTest( + "select province,city,region,device_id,count(s1), count(distinct s1) from table1 group by 1,2,3,4 order by 1,2,3,4", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void singleInputDistinctAggregationTest() { + String[] expectedHeader = new String[] {"_col0", "_col1"}; + String[] retArray = + new String[] { + "5,40.4,", + }; + + tableResultSetEqualTest( + "select count(distinct s1), avg(distinct s1) from table1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"16,"}; + tableResultSetEqualTest( + "select count(distinct device_id) from table1 group by date_bin(1d,time) order by 1", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"province", "city", "region", "_col3", "_col4"}; + retArray = + new String[] { + "beijing,beijing,chaoyang,5,40.4,", + "beijing,beijing,haidian,5,40.4,", + "shanghai,shanghai,huangpu,5,40.4,", + "shanghai,shanghai,pudong,5,40.4," + }; + tableResultSetEqualTest( + "select province,city,region,count(distinct s1), avg(distinct s1) from table1 group by 1,2,3 order by 1,2,3", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void exceptionTest2() { + tableAssertTestFail( + "select count(distinct *) from table1", + "mismatched input '*'. Expecting: ", + DATABASE_NAME); + + String errMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": Unsupported expression: Row"; + tableAssertTestFail("select distinct (s1,s2) from table1", errMsg, DATABASE_NAME); + + tableAssertTestFail("select (s1,s2) from table1", errMsg, DATABASE_NAME); + + tableAssertTestFail("select * from table1 where (s1,s2) is not null", errMsg, DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java new file mode 100644 index 0000000000000..17a5cf9ad0056 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.view.recent; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.apache.tsfile.read.common.RowRecord; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; + +@RunWith(Parameterized.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableViewQueryIT { + + protected static final String DATABASE_NAME = "test"; + + protected static String[] createTreeAlignedDataSqls = { + "CREATE ALIGNED TIMESERIES root.db.battery.b1(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (1, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (2, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (3, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (4, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (5, 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b2(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b2(time, voltage, current) aligned values (1, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) aligned values (2, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) aligned values (3, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) aligned values (4, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) aligned values (5, null, 2)", + "CREATE ALIGNED TIMESERIES root.db.battery.b3(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b3(time, voltage, current) aligned values (1, 3, 3)", + "INSERT INTO root.db.battery.b3(time, voltage, current) aligned values (2, null, 3)", + "INSERT INTO root.db.battery.b3(time, voltage, current) aligned values (3, 3, 3)", + "INSERT INTO root.db.battery.b3(time, voltage, current) aligned values (4, 3, 3)", + "CREATE ALIGNED TIMESERIES root.db.battery.b4(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b4(time, voltage, current) aligned values (1, 4, 4)", + "INSERT INTO root.db.battery.b4(time, voltage, current) aligned values (2, 4, 4)", + "INSERT INTO root.db.battery.b4(time, voltage, current) aligned values (3, 4, null)", + "INSERT INTO root.db.battery.b4(time, voltage, current) aligned values (4, 4, 4)", + }; + protected static String[] createTreeNonAlignedDataSqls = { + "CREATE TIMESERIES root.db.battery.b1.voltage INT32", + "CREATE TIMESERIES root.db.battery.b1.current FLOAT", + "INSERT INTO root.db.battery.b1(time, voltage, current) values (1, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) values (2, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) values (3, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) values (4, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) values (5, 1, 1)", + "CREATE TIMESERIES root.db.battery.b2.voltage INT32", + "CREATE TIMESERIES root.db.battery.b2.current FLOAT", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (1, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (2, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (3, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (4, 2, 2)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (5, null, 2)", + "CREATE TIMESERIES root.db.battery.b3.voltage INT32", + "CREATE TIMESERIES root.db.battery.b3.current FLOAT", + "INSERT INTO root.db.battery.b3(time, voltage, current) values (1, 3, 3)", + "INSERT INTO root.db.battery.b3(time, voltage, current) values (2, null, 3)", + "INSERT INTO root.db.battery.b3(time, voltage, current) values (3, 3, 3)", + "INSERT INTO root.db.battery.b3(time, voltage, current) values (4, 3, 3)", + "CREATE TIMESERIES root.db.battery.b4.voltage INT32", + "CREATE TIMESERIES root.db.battery.b4.current FLOAT", + "INSERT INTO root.db.battery.b4(time, voltage, current) values (1, 4, 4)", + "INSERT INTO root.db.battery.b4(time, voltage, current) values (2, 4, 4)", + "INSERT INTO root.db.battery.b4(time, voltage, current) values (3, 4, null)", + "INSERT INTO root.db.battery.b4(time, voltage, current) values (4, 4, 4)", + }; + + protected static String[] createTableSqls = { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE VIEW view1 (battery TAG, voltage INT32 FIELD, current FLOAT FIELD) as root.db.battery.**", + "CREATE VIEW view2 (battery TAG, voltage INT32 FIELD FROM voltage, current_rename FLOAT FIELD FROM current) as root.db.battery.**", + "CREATE VIEW view3 (battery TAG, voltage INT32 FIELD FROM voltage, current_rename FLOAT FIELD FROM current) with (ttl=1) as root.db.battery.**", + "CREATE TABLE table1 (battery TAG, voltage INT32 FIELD, current FLOAT FIELD)", + "INSERT INTO table1 (time, battery, voltage, current) values (1, 'b1', 1, 1)", + "INSERT INTO table1 (time, battery, voltage, current) values (2, 'b1', 1, 1)", + "INSERT INTO table1 (time, battery, voltage, current) values (3, 'b1', 1, 1)", + "INSERT INTO table1 (time, battery, voltage, current) values (4, 'b1', 1, 1)", + "INSERT INTO table1 (time, battery, voltage, current) values (5, 'b1', 1, 1)", + "INSERT INTO table1 (time, battery, voltage, current) values (1, 'b2', 2, 2)", + "INSERT INTO table1 (time, battery, voltage, current) values (2, 'b2', 2, 2)", + "INSERT INTO table1 (time, battery, voltage, current) values (3, 'b2', 2, 2)", + "INSERT INTO table1 (time, battery, voltage, current) values (4, 'b2', 2, 2)", + "INSERT INTO table1 (time, battery, voltage, current) values (5, 'b2', null, 2)", + "INSERT INTO table1 (time, battery, voltage, current) values (1, 'b3', 3, 3)", + "INSERT INTO table1 (time, battery, voltage, current) values (2, 'b3', null, 3)", + "INSERT INTO table1 (time, battery, voltage, current) values (3, 'b3', 3, 3)", + "INSERT INTO table1 (time, battery, voltage, current) values (4, 'b3', 3, 3)", + "INSERT INTO table1 (time, battery, voltage, current) values (1, 'b4', 4, 4)", + "INSERT INTO table1 (time, battery, voltage, current) values (2, 'b4', 4, 4)", + "INSERT INTO table1 (time, battery, voltage, current) values (3, 'b4', 4, null)", + "INSERT INTO table1 (time, battery, voltage, current) values (4, 'b4', 4, 4)", + }; + + @Parameterized.Parameters(name = "aligned={0}, flush={1}") + public static Collection data() { + return Arrays.asList( + new Object[][] {{true, true}, {false, true}, {true, false}, {false, false}}); + } + + private final boolean aligned; + private final boolean flush; + + public IoTDBTableViewQueryIT(boolean aligned, boolean flush) { + this.aligned = aligned; + this.flush = flush; + } + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(aligned ? createTreeAlignedDataSqls : createTreeNonAlignedDataSqls); + if (flush) { + prepareData(new String[] {"flush"}); + } + prepareTableData(createTableSqls); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + compareQueryResults(session, "select * from view1", "select * from table1", true); + compareQueryResults( + session, + "select time, battery, current, voltage from view1", + "select time, battery, current_rename, voltage from view2", + true); + compareQueryResults(session, "select battery from view1", "select battery from table1", true); + compareQueryResults( + session, + "select current from view1 where time > 1", + "select current from table1 where time > 1" + (aligned ? "" : " and current is not null"), + true); + compareQueryResults( + session, + "select * from view1 fill method linear", + "select * from table1 fill method linear", + true); + compareQueryResults( + session, + "select * from view1 fill method previous", + "select * from table1 fill method previous", + true); + compareQueryResults( + session, + "select * from view1 fill method constant 0", + "select * from table1 fill method constant 0", + true); + compareQueryResults( + session, + "select * from view1 where time > 1", + "select * from table1 where time > 1", + true); + compareQueryResults( + session, + "select * from view1 where time > 1 and voltage > 1", + "select * from table1 where time > 1 and voltage > 1", + true); + compareQueryResults( + session, + "select * from view1 where time > 1 and (voltage + current) > 1", + "select * from table1 where time > 1 and (voltage + current) > 1", + true); + compareQueryResults( + session, + "select * from view1 where voltage is null", + "select * from table1 where voltage is null", + true); + compareQueryResults( + session, + "select * from view1 where current is not null", + "select * from table1 where current is not null", + true); + compareQueryResults( + session, + "select * from view1 where current > 1 and voltage > 3", + "select * from table1 where current > 1 and voltage > 3", + true); + compareQueryResults( + session, + "select * from view1 where current > 1 or voltage > 3", + "select * from table1 where current > 1 or voltage > 3", + true); + compareQueryResults( + session, + "select * from view1 where battery='b1' limit 1", + "select * from table1 where battery='b1' limit 1", + true); + compareQueryResults( + session, + "select * from view1 where battery='b1' offset 1 limit 1", + "select * from table1 where battery='b1' offset 1 limit 1", + true); + compareQueryResults( + session, + "select * from view1 where battery='b1' offset 5 limit 1", + "select * from table1 where battery='b1' offset 5 limit 1", + true); + compareQueryResults( + session, + "select * from view1 order by battery, time limit 1", + "select * from table1 order by battery, time limit 1", + false); + compareQueryResults( + session, + "select * from view1 order by battery, time offset 2 limit 2", + "select * from table1 order by battery, time offset 2 limit 2", + false); + compareQueryResults( + session, + "select * from view1 order by battery, time asc limit 1", + "select * from table1 order by battery, time limit 1", + false); + compareQueryResults( + session, + "select * from view1 order by battery, time asc offset 2 limit 2", + "select * from table1 order by battery, time offset 2 limit 2", + false); + + compareQueryResults( + session, + "select time, battery, current from view1", + "select time, battery, current from table1" + + (aligned ? "" : " where current is not null"), + true); + compareQueryResults( + session, + "select time, battery, current from view1 where voltage > 1", + "select time, battery, current from table1 where voltage > 1", + true); + compareQueryResults( + session, + "select time, battery, current from view1 where voltage > 1 and current > 3", + "select time, battery, current from table1 where voltage > 1 and current > 3", + true); + compareQueryResults( + session, + "select time, battery, current from view1 where voltage > 1 or current > 0", + "select time, battery, current from table1 where voltage > 1 or current > 0", + true); + + compareQueryResults( + session, "select count(*) from view1", "select count(*) from table1", true); + compareQueryResults( + session, "select count(battery) from view1", "select count(battery) from table1", true); + compareQueryResults( + session, + "select count(*) from view1 where time = 1", + "select count(*) from table1 where time = 1", + true); + compareQueryResults( + session, + "select count(*) from view1 group by battery", + "select count(*) from table1 group by battery", + true); + compareQueryResults( + session, + "select count(*) from view1 where time > 1 group by battery", + "select count(*) from table1 where time > 1 group by battery", + true); + + compareQueryResults( + session, "select avg(current) from view1", "select avg(current) from table1", true); + compareQueryResults( + session, + "select count(*) from view1 where time = 1", + "select count(*) from table1 where time = 1", + true); + compareQueryResults( + session, + "select count(*) from view1 group by battery", + "select count(*) from table1 group by battery", + true); + compareQueryResults( + session, + "select count(*) from view1 where time > 1 group by battery", + "select count(*) from table1 where time > 1 group by battery", + true); + compareQueryResults( + session, + "select count(*) from view1 group by battery having count(*) >= 5", + "select count(*) from table1 group by battery having count(*) >= 5", + true); + + compareQueryResults( + session, + "select current from view1 where current >= (select avg(current) from view1)", + "select current from table1 where current >= (select avg(current) from table1)", + true); + + compareQueryResults( + session, + "select current from view1 where battery='b1' and current >= (select avg(current) from view1 where battery='b1')", + "select current from table1 where battery='b1' and current >= (select avg(current) from table1 where battery='b1')", + true); + // empty result + compareQueryResults( + session, "select * from view3 limit 1", "select * from table1 limit 0", true); + } + } + + private static void compareQueryResults( + ITableSession session, String sql1, String sql2, boolean sort) throws Exception { + List records1 = new ArrayList<>(); + List records2 = new ArrayList<>(); + session.executeNonQueryStatement("USE " + DATABASE_NAME); + SessionDataSet sessionDataSet = session.executeQueryStatement(sql1); + while (sessionDataSet.hasNext()) { + RowRecord record = sessionDataSet.next(); + records1.add(record.toString()); + } + sessionDataSet.close(); + sessionDataSet = session.executeQueryStatement(sql2); + while (sessionDataSet.hasNext()) { + RowRecord record = sessionDataSet.next(); + records2.add(record.toString()); + } + sessionDataSet.close(); + if (sort) { + records1.sort(String::compareTo); + records2.sort(String::compareTo); + } + Assert.assertEquals(records1, records2); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceCaseWhenThenIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceCaseWhenThenIT.java new file mode 100644 index 0000000000000..dfba014cde7cf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceCaseWhenThenIT.java @@ -0,0 +1,491 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.rest.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRestServiceCaseWhenThenIT { + + private int port = 18080; + private CloseableHttpClient httpClient = null; + + @Before + public void setUp() throws Exception { + BaseEnv baseEnv = EnvFactory.getEnv(); + baseEnv.getConfig().getDataNodeConfig().setEnableRestService(true); + baseEnv.initClusterEnvironment(); + DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + port = portConflictDataNodeWrapper.getRestServicePort(); + httpClient = HttpClientBuilder.create().build(); + } + + @After + public void tearDown() throws Exception { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static final String DATABASE = "test"; + + private static final String[] expectedHeader = {"_col0"}; + + private static final String[] SQLs = + new String[] { + // normal cases + "CREATE DATABASE " + DATABASE, + "CREATE table table1 (device_id STRING TAG, s1 INT32 FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD)", + "CREATE table table2 (device_id STRING TAG, s3 FLOAT FIELD, s4 DOUBLE FIELD)", + "CREATE table table3 (device_id STRING TAG, s2 INT64 FIELD)", + "INSERT INTO table1(time, device_id, s1) values(100, 'd1', 0)", + "INSERT INTO table1(time, device_id, s1) values(200, 'd1', 11)", + "INSERT INTO table1(time, device_id, s1) values(300, 'd1', 22)", + "INSERT INTO table1(time, device_id, s1) values(400, 'd1', 33)", + "INSERT INTO table2(time, device_id, s3) values(100, 'd1', 0)", + "INSERT INTO table2(time, device_id, s3) values(200, 'd1', 11)", + "INSERT INTO table2(time, device_id, s3) values(300, 'd1', 22)", + "INSERT INTO table2(time, device_id, s3) values(400, 'd1', 33)", + "INSERT INTO table2(time, device_id, s4) values(100, 'd1', 44)", + "INSERT INTO table2(time, device_id, s4) values(200, 'd1', 55)", + "INSERT INTO table2(time, device_id, s4) values(300, 'd1', 66)", + "INSERT INTO table2(time, device_id, s4) values(400, 'd1', 77)", + }; + + @Test + public void test() { + ping(); + prepareTableData(); + testKind1Basic(); + testKind2Basic(); + testShortCircuitEvaluation(); + testKind1InputTypeRestrict(); + testKind2InputTypeRestrict(); + testKind1OutputTypeRestrict(); + testKind2OutputTypeRestrict(); + testKind1UsedInOtherOperation(); + testKind2UsedInOtherOperation(); + testKind1UseOtherOperation(); + testKind2UseOtherOperation(); + testKind1UseInWhereClause(); + testKind1CaseInCase(); + testKind2CaseInCase(); + testKind1Logic(); + testKind2UseOtherOperation(); + testKind1UseInWhereClause(); + } + + public void ping() { + HttpGet httpGet = new HttpGet("http://127.0.0.1:" + port + "/ping"); + CloseableHttpResponse response = null; + try { + for (int i = 0; i < 30; i++) { + try { + response = httpClient.execute(httpGet); + break; + } catch (Exception e) { + if (i == 29) { + throw e; + } + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + JsonObject result = JsonParser.parseString(message).getAsJsonObject(); + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(200, Integer.parseInt(result.get("code").toString())); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + } + + public void testKind1Basic() { + String[] retArray = new String[] {"99,", "9999,", "9999,", "999,"}; + tableResultSetEqualTest( + "select case when s1=0 then 99 when s1>22 then 999 else 9999 end from table1", + expectedHeader, + retArray, + DATABASE); + } + + public void testKind2Basic() { + String sql = "select case s1 when 0 then 99 when 22 then 999 else 9999 end from table1"; + String[] retArray = new String[] {"99,", "9999,", "999,", "9999,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // without ELSE clause + sql = "select case s1 when 0 then 99 when 22 then 999 end from table1"; + retArray = new String[] {"99,", "null,", "999,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testShortCircuitEvaluation() { + String[] retArray = new String[] {"0,", "11,", "22,", "33,"}; + tableResultSetEqualTest( + "select case when 1=0 then s1/0 when 1!=0 then s1 end from table1", + expectedHeader, + retArray, + DATABASE); + } + + public void testKind1InputTypeRestrict() { + // WHEN clause must return BOOLEAN + String sql = "select case when s1+1 then 20 else 22 end from table1"; + String msg = "701: CASE WHEN clause must evaluate to a BOOLEAN (actual: INT32)"; + tableAssertTestFail(sql, msg, DATABASE); + } + + public void testKind2InputTypeRestrict() { + // the expression in CASE clause must be able to be equated with the expression in WHEN clause + String sql = "select case s1 when '1' then 20 else 22 end from table1"; + String msg = "701: CASE operand type does not match WHEN clause operand type: INT32 vs STRING"; + tableAssertTestFail(sql, msg, DATABASE); + } + + public void testKind1OutputTypeRestrict() { + // BOOLEAN and other types cannot exist at the same time + String[] retArray = new String[] {"true,", "false,", "true,", "true,"}; + // success + tableResultSetEqualTest( + "select case when s1<=0 then true when s1=11 then false else true end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case when s1<=0 then true else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between BOOLEAN and INT32, all types (without duplicates): [BOOLEAN, INT32]", + DATABASE); + + // TEXT and other types cannot exist at the same time + retArray = new String[] {"good,", "bad,", "okok,", "okok,"}; + // success + tableResultSetEqualTest( + "select case when s1<=0 then 'good' when s1=11 then 'bad' else 'okok' end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case when s1<=0 then 'good' else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between STRING and INT32, all types (without duplicates): [STRING, INT32]", + DATABASE); + } + + public void testKind2OutputTypeRestrict() { + // BOOLEAN and other types cannot exist at the same time + String[] retArray = + new String[] { + "true,", "false,", "true,", "true,", + }; + // success + tableResultSetEqualTest( + "select case s1 when 0 then true when 11 then false else true end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case s1 when 0 then true else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between BOOLEAN and INT32, all types (without duplicates): [BOOLEAN, INT32]", + DATABASE); + + // TEXT and other types cannot exist at the same time + retArray = new String[] {"good,", "bad,", "okok,", "okok,"}; + // success + tableResultSetEqualTest( + "select case s1 when 0 then 'good' when 11 then 'bad' else 'okok' end from table1", + expectedHeader, + retArray, + DATABASE); + // fail + tableAssertTestFail( + "select case s1 when 0 then 'good' else 22 end from table1", + "701: All CASE results must be the same type or coercible to a common type. Cannot find common type between STRING and INT32, all types (without duplicates): [STRING, INT32]", + DATABASE); + } + + public void testKind1UsedInOtherOperation() { + String sql; + String[] retArray; + + // use in scalar operation + + // multiply + sql = "select 2 * case when s1=0 then 99 when s1=22.0 then 999 else 9999 end from table1"; + retArray = new String[] {"198,", "19998,", "1998,", "19998,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // add + sql = + "select " + + "case when s1=0 then 99 when s1=22.0 then 999 else 9999 end " + + "+" + + "case when s1=11 then 99 else 9999 end " + + "from table1"; + retArray = + new String[] { + "10098,", "10098,", "10998,", "19998,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // function + sql = "select diff(case when s1=0 then 99 when s1>22 then 999 else 9999 end) from table1"; + retArray = new String[] {"null,", "9900.0,", "0.0,", "-9000.0,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testKind2UsedInOtherOperation() { + String sql; + String[] retArray; + + // use in scalar operation + + // multiply + sql = "select 2 * case s1 when 0 then 99 when 22 then 999 else 9999 end from table1"; + retArray = new String[] {"198,", "19998,", "1998,", "19998,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + sql = "select diff(case s1 when 0 then 99 when 22 then 999 else 9999 end) from table1"; + retArray = + new String[] { + "null,", "9900.0,", "-9000.0,", "9000.0,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testKind1UseOtherOperation() { + // WHEN-clause use scalar function + String sql = "select case when sin(s1)>=0 then '>0' else '<0' end from table1"; + String[] retArray = + new String[] { + ">0,", "<0,", "<0,", ">0,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // THEN-clause and ELSE-clause use scalar function + + // TODO: align by is not supported. + + // sql = + // "select case when s1<=11 then CAST(diff(s1) as TEXT) else CAST(s1-1 as TEXT) end from + // table1 align by device"; + // + // retArray = + // new String[] { + // "0,table1,null,", + // "1000000,table1,11.0,", + // "20000000,table1,21.0,", + // "210000000,table1,32.0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testKind2UseOtherOperation() { + // CASE-clause use scalar function + String sql = + "select case round(sin(s1)) when 0 then '=0' when -1 then '<0' else '>0' end from table1"; + + tableAssertTestFail( + sql, + "701: CASE operand type does not match WHEN clause operand type: DOUBLE vs INT32", + DATABASE); + // String[] retArray = + // new String[] { + // "=0,", "<0,", ">0,", ">0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // WHEN-clause use scalar function + sql = "select case 0 when sin(s1) then '=0' else '!=0' end from table1"; + tableAssertTestFail( + sql, + "701: CASE operand type does not match WHEN clause operand type: INT32 vs DOUBLE", + DATABASE); + // retArray = + // new String[] { + // "=0,", "!=0,", "!=0,", "!=0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // THEN-clause and ELSE-clause use scalar function + // sql = + // "select case s1 when 11 then CAST(diff(s1) as TEXT) else CAST(s1-1 as TEXT) end from + // table1 align by device"; + // + // retArray = + // new String[] { + // "table1,-1.0,", "table1,11.0,", "table1,21.0,", "table1,32.0,", + // }; + // tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + + // UDF is not allowed + // sql = "select case s1 when 0 then change_points(s1) end from table1"; + // String msg = "301: CASE expression cannot be used with non-mappable UDF"; + // tableAssertTestFail(sql, msg, DATABASE); + } + + public void testKind1UseInWhereClause() { + String sql = + "select s4 from table2 where case when s3=0 then s4>44 when s3=22 then s4>0 when time>300 then true end"; + String[] retArray = new String[] {"66.0,", "77.0,"}; + tableResultSetEqualTest(sql, new String[] {"s4"}, retArray, DATABASE); + + sql = + "select case when s3=0 then s4>44 when s3=22 then s4>0 when time>300 then true end from table2"; + retArray = + new String[] { + "false,", "null,", "true,", "true,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testKind1CaseInCase() { + String sql = + "select case when s1=0 OR s1=22 then cast(case when s1=0 then 99 when s1>22 then 999 end as STRING) else 'xxx' end from table1"; + String[] retArray = + new String[] { + "99,", "xxx,", "null,", "xxx,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testKind2CaseInCase() { + String sql = + "select case s1 when 0 then cast(case when s1=0 then 99 when s1>22 then 999 end as STRING) when 22 then cast(case when s1=0 then 99 when s1>22 then 999 end as STRING) else 'xxx' end from table1"; + String[] retArray = + new String[] { + "99,", "xxx,", "null,", "xxx,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void testKind1Logic() { + String sql = + "select case when s3 >= 0 and s3 < 20 and s4 >= 50 and s4 < 60 then 'just so so~~~' when s3 >= 20 and s3 < 40 and s4 >= 70 and s4 < 80 then 'very well~~~' end from table2"; + String[] retArray = new String[] {"null,", "just so so~~~,", "null,", "very well~~~,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); + } + + public void tableAssertTestFail(String sql, String errMsg, String database) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("database", database); + jsonObject.addProperty("sql", sql); + JsonObject result = query(jsonObject.toString()); + assertEquals(errMsg, result.get("code") + ": " + result.get("message").getAsString()); + } + + public void tableResultSetEqualTest( + String sql, String[] expectedHeader, String[] expectedRetArray, String database) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("database", database); + jsonObject.addProperty("sql", sql); + JsonObject result = query(jsonObject.toString()); + JsonArray columnNames = result.get("column_names").getAsJsonArray(); + JsonArray valuesList = result.get("values").getAsJsonArray(); + for (int i = 0; i < columnNames.size(); i++) { + assertEquals(expectedHeader[i], columnNames.get(i).getAsString()); + } + assertEquals(expectedHeader.length, columnNames.size()); + int cnt = 0; + for (int i = 0; i < valuesList.size(); i++) { + StringBuilder builder = new StringBuilder(); + JsonArray values = valuesList.get(i).getAsJsonArray(); + for (int c = 0; c < values.size(); c++) { + if (!values.get(c).isJsonNull()) { + builder.append(values.get(c).getAsString()).append(","); + } else { + builder.append(values.get(c).toString()).append(","); + } + } + assertEquals(expectedRetArray[i], builder.toString()); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + + public void prepareTableData() { + for (int i = 0; i < SQLs.length; i++) { + JsonObject jsonObject = new JsonObject(); + if (i > 0) { + jsonObject.addProperty("database", DATABASE); + } else { + jsonObject.addProperty("database", ""); + } + jsonObject.addProperty("sql", SQLs[i]); + nonQuery(jsonObject.toString()); + } + } + + public JsonObject query(String json) { + return RestUtils.query(httpClient, port, json); + } + + public JsonObject nonQuery(String json) { + return RestUtils.nonQuery(httpClient, port, json); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceFlushQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceFlushQueryIT.java new file mode 100644 index 0000000000000..9ec47467ec608 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceFlushQueryIT.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.rest.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.*; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRestServiceFlushQueryIT { + + private int port = 18080; + private CloseableHttpClient httpClient = null; + + @Before + public void setUp() throws Exception { + BaseEnv baseEnv = EnvFactory.getEnv(); + baseEnv.getConfig().getDataNodeConfig().setEnableRestService(true); + baseEnv.initClusterEnvironment(); + DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + port = portConflictDataNodeWrapper.getRestServicePort(); + httpClient = HttpClientBuilder.create().build(); + } + + @After + public void tearDown() throws Exception { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static final String DATABASE = "test"; + + private static final String[] sqls = + new String[] { + "CREATE DATABASE test", + "CREATE TABLE vehicle (tag1 string tag, s0 int32 field)", + "insert into vehicle(tag1,time,s0) values('d0',1,101)", + "insert into vehicle(tag1,time,s0) values('d0',2,198)", + "insert into vehicle(tag1,time,s0) values('d0',100,99)", + "insert into vehicle(tag1,time,s0) values('d0',101,99)", + "insert into vehicle(tag1,time,s0) values('d0',102,80)", + "insert into vehicle(tag1,time,s0) values('d0',103,99)", + "insert into vehicle(tag1,time,s0) values('d0',104,90)", + "insert into vehicle(tag1,time,s0) values('d0',105,99)", + "insert into vehicle(tag1,time,s0) values('d0',106,99)", + "flush", + "insert into vehicle(tag1,time,s0) values('d0',2,10000)", + "insert into vehicle(tag1,time,s0) values('d0',50,10000)", + "insert into vehicle(tag1,time,s0) values('d0',1000,22222)", + }; + + public void ping() { + HttpGet httpGet = new HttpGet("http://127.0.0.1:" + port + "/ping"); + CloseableHttpResponse response = null; + try { + for (int i = 0; i < 30; i++) { + try { + response = httpClient.execute(httpGet); + break; + } catch (Exception e) { + if (i == 29) { + throw e; + } + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + JsonObject result = JsonParser.parseString(message).getAsJsonObject(); + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(200, Integer.parseInt(result.get("code").toString())); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + } + + @Test + public void test() { + ping(); + prepareTableData(); + selectAllSQLTest(); + testFlushGivenGroup(); + testFlushGivenGroupNoData(); + } + + public String sqlHandler(String database, String sql) { + JsonObject json = new JsonObject(); + json.addProperty("database", database); + json.addProperty("sql", sql); + return json.toString(); + } + + public void selectAllSQLTest() { + String sql = sqlHandler("test", "SELECT * FROM vehicle"); + JsonObject jsonObject = query(sql); + JsonArray valuesList = jsonObject.getAsJsonArray("values"); + for (int i = 0; i < valuesList.size(); i++) { + JsonArray jsonArray = valuesList.get(i).getAsJsonArray(); + for (int j = 0; j < jsonArray.size(); j++) { + jsonArray.get(j); + } + } + } + + public void testFlushGivenGroup() { + List list = + Arrays.asList("CREATE DATABASE group1", "CREATE DATABASE group2", "CREATE DATABASE group3"); + for (String sql : list) { + nonQuery(sqlHandler("", sql)); + } + + String insertTemplate = + "INSERT INTO vehicle(tag1, time, s1, s2, s3) VALUES (%s, %d, %d, %f, %s)"; + for (int i = 1; i <= 3; i++) { + nonQuery(sqlHandler("", String.format("USE \"group%d\"", i))); + nonQuery( + sqlHandler( + String.format("group%d", i), + "CREATE TABLE vehicle (tag1 string tag, s1 int32 field, s2 float field, s3 string field)")); + + for (int j = 10; j < 20; j++) { + nonQuery(String.format(Locale.CHINA, insertTemplate, i, j, j, j * 0.1, j)); + } + } + nonQuery(sqlHandler("", "FLUSH")); + + for (int i = 1; i <= 3; i++) { + nonQuery(sqlHandler("", String.format("USE \"group%d\"", i))); + nonQuery( + sqlHandler( + String.format("group%d", i), + "CREATE TABLE vehicle (tag1 string tag, s1 int32 field, s2 float field, s3 string field)")); + } + nonQuery(sqlHandler("", "FLUSH group1")); + nonQuery(sqlHandler("", "FLUSH group2,group3")); + + for (int i = 1; i <= 3; i++) { + nonQuery(sqlHandler("", String.format("USE \"group%d\"", i))); + nonQuery( + sqlHandler( + String.format("group%d", i), + "CREATE TABLE vehicle (tag1 string tag, s1 int32 field, s2 float field, s3 string field)")); + + for (int j = 0; j < 30; j++) { + nonQuery(String.format(Locale.CHINA, insertTemplate, i, j, j, j * 0.1, j)); + } + } + nonQuery(sqlHandler("", "FLUSH group1 TRUE")); + nonQuery(sqlHandler("", "FLUSH group2,group3 FALSE")); + + for (int i = 1; i <= 3; i++) { + int count = 0; + nonQuery(sqlHandler("", String.format("USE \"group%d\"", i))); + JsonObject jsonObject = + query(sqlHandler(String.format("group%d", i), "SELECT * FROM vehicle")); + + JsonArray valuesList = jsonObject.getAsJsonArray("values"); + for (int c = 0; c < valuesList.size(); c++) { + count++; + JsonArray jsonArray = valuesList.get(c).getAsJsonArray(); + for (int j = 0; j < jsonArray.size(); j++) { + jsonArray.get(j); + } + assertEquals(30, count); + } + } + } + + public void testFlushGivenGroupNoData() { + List list = + Arrays.asList( + "CREATE DATABASE nodatagroup1", + "CREATE DATABASE nodatagroup2", + "CREATE DATABASE nodatagroup3", + "FLUSH nodatagroup1", + "FLUSH nodatagroup2", + "FLUSH nodatagroup3", + "FLUSH nodatagroup1, nodatagroup2"); + for (String sql : list) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("database", ""); + jsonObject.addProperty("sql", sql); + nonQuery(jsonObject.toString()); + } + } + + public void tableAssertTestFail(String sql, String errMsg, String database) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("database", database); + jsonObject.addProperty("sql", sql); + JsonObject result = query(jsonObject.toString()); + assertEquals(errMsg, result.get("code") + ": " + result.get("message").getAsString()); + } + + public void tableResultSetEqualTest( + String sql, String[] expectedHeader, String[] expectedRetArray, String database) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("database", database); + jsonObject.addProperty("sql", sql); + JsonObject result = query(jsonObject.toString()); + JsonArray columnNames = result.get("column_names").getAsJsonArray(); + JsonArray valuesList = result.get("values").getAsJsonArray(); + for (int i = 0; i < columnNames.size(); i++) { + assertEquals(expectedHeader[i], columnNames.get(i).getAsString()); + } + assertEquals(expectedHeader.length, columnNames.size()); + int cnt = 0; + for (int i = 0; i < valuesList.size(); i++) { + StringBuilder builder = new StringBuilder(); + JsonArray values = valuesList.get(i).getAsJsonArray(); + for (int c = 0; c < values.size(); c++) { + if (!values.get(c).isJsonNull()) { + builder.append(values.get(c).getAsString()).append(","); + } else { + builder.append(values.get(c).toString()).append(","); + } + } + assertEquals(expectedRetArray[i], builder.toString()); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + + public void prepareTableData() { + for (int i = 0; i < sqls.length; i++) { + JsonObject jsonObject = new JsonObject(); + if (i > 0) { + jsonObject.addProperty("database", DATABASE); + } else { + jsonObject.addProperty("database", ""); + } + jsonObject.addProperty("sql", sqls[i]); + nonQuery(jsonObject.toString()); + } + } + + public JsonObject query(String json) { + return RestUtils.query(httpClient, port, json); + } + + public JsonObject nonQuery(String json) { + return RestUtils.nonQuery(httpClient, port, json); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceIT.java new file mode 100644 index 0000000000000..156ecdeed321f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceIT.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.rest.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRestServiceIT { + + private int port = 18080; + private CloseableHttpClient httpClient = null; + + @Before + public void setUp() throws Exception { + BaseEnv baseEnv = EnvFactory.getEnv(); + baseEnv.getConfig().getDataNodeConfig().setEnableRestService(true); + baseEnv.initClusterEnvironment(); + DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + port = portConflictDataNodeWrapper.getRestServicePort(); + httpClient = HttpClientBuilder.create().build(); + } + + @After + public void tearDown() throws Exception { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static final String DATABASE = "test"; + + private static final String[] sqls = + new String[] { + "create database if not exists test", + "use test", + "CREATE TABLE sg10(tag1 string tag, s1 int64 field, s2 float field, s3 string field)", + "CREATE TABLE sg11(tag1 string tag, s1 int64 field, s2 float field, s3 string field)" + }; + + public void ping() { + HttpGet httpGet = new HttpGet("http://127.0.0.1:" + port + "/ping"); + CloseableHttpResponse response = null; + try { + for (int i = 0; i < 30; i++) { + try { + response = httpClient.execute(httpGet); + break; + } catch (Exception e) { + if (i == 29) { + throw e; + } + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + JsonObject result = JsonParser.parseString(message).getAsJsonObject(); + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(200, Integer.parseInt(result.get("code").toString())); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + } + + @Test + public void test() { + ping(); + prepareTableData(); + rightNonQuery(); + rightNonQuery2(); + rightNonQuery3(); + rightNonQuery4(); + errorNonQuery(); + errorNonQuery1(); + errorNonQuery3(); + testInsertMultiPartition(); + testInsertTablet(); + testInsertTabletNoDatabase(); + testInsertTablet1(); + testInsertTablet2(); + testQuery(); + testQuery1(); + testQuery2(); + } + + public void testQuery() { + String sql = "insert into sg11(tag1,s1,s2,s3,time) values('aa',11,1.1,1,1),('aa2',21,2.1,2,2)"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + assertEquals(200, result.get("code").getAsInt()); + JsonObject queryResult = + RestUtils.query( + httpClient, + port, + sqlHandler("test", "select tag1,s1,s2,s3,time from sg11 order by time")); + JsonArray jsonArray = queryResult.get("values").getAsJsonArray(); + for (int i = 0; i < jsonArray.size(); i++) { + JsonArray jsonArray1 = jsonArray.get(i).getAsJsonArray(); + + if (i == 0) { + assertEquals("aa", jsonArray1.get(0).getAsString()); + assertEquals(11, jsonArray1.get(1).getAsInt()); + assertEquals(1.1, jsonArray1.get(2).getAsFloat(), 0.000001f); + assertEquals("1", jsonArray1.get(3).getAsString()); + } else if (i == 1) { + assertEquals("aa2", jsonArray1.get(0).getAsString()); + assertEquals(21, jsonArray1.get(1).getAsInt()); + assertEquals(2.1, jsonArray1.get(2).getAsFloat(), 0.000001f); + assertEquals("2", jsonArray1.get(3).getAsString()); + assertEquals(2, jsonArray1.get(4).getAsLong()); + } + } + } + + public void testQuery1() { + JsonObject result = + RestUtils.query( + httpClient, + port, + sqlHandler(null, "select tag1,s1,s2,s3,time from sg11 order by time")); + assertEquals(701, result.get("code").getAsInt()); + assertEquals( + "Database must be specified when session database is not set", + result.get("message").getAsString()); + } + + public void testQuery2() { + JsonObject result = RestUtils.query(httpClient, port, sqlHandler("test", null)); + assertEquals(305, result.get("code").getAsInt()); + assertEquals("sql should not be null", result.get("message").getAsString()); + } + + public void rightNonQuery() { + String sql = "create database test1"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("", sql)); + assertEquals(200, result.get("code").getAsInt()); + } + + public void rightNonQuery2() { + String sql = "insert into sg10(tag1,s1,time,s2) values('aa',1,1,1.1)"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + assertEquals(200, result.get("code").getAsInt()); + } + + public void rightNonQuery4() { + String sql = "insert into sg10(tag1,s1,time,s2) values('aa',1,1,1.1),('bb',2,2,2.1)"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + assertEquals(200, result.get("code").getAsInt()); + } + + public void rightNonQuery3() { + String sql = "drop database test1"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + assertEquals(200, result.get("code").getAsInt()); + } + + public void errorNonQuery() { + String sql = "create database test"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("", sql)); + assertEquals(501, result.get("code").getAsInt()); + assertEquals("Database test already exists", result.get("message").getAsString()); + } + + public void errorNonQuery1() { + String sql = + "CREATE TABLE sg10(tag1 string tag, s1 int64 field, s2 float field, s3 string field)"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler(null, sql)); + assertEquals(701, result.get("code").getAsInt()); + assertEquals("database is not specified", result.get("message").getAsString()); + } + + public void errorNonQuery2() { + String sql = "create database test"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler(null, sql)); + assertEquals(701, result.get("code").getAsInt()); + assertEquals("database is not specified", result.get("message").getAsString()); + } + + public void errorNonQuery3() { + String sql = "select * from sg10"; + JsonObject result = RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + assertEquals(301, result.get("code").getAsInt()); + assertEquals("EXECUTE_STATEMENT_ERROR", result.get("message").getAsString()); + } + + public String sqlHandler(String database, String sql) { + JsonObject json = new JsonObject(); + json.addProperty("database", database); + json.addProperty("sql", sql); + return json.toString(); + } + + public void testInsertMultiPartition() { + List sqls = + Arrays.asList( + "create table sg1 (tag1 string tag, s1 int32 field)", + "insert into sg1(tag1,time,s1) values('d1',1,2)", + "flush", + "insert into sg1(tag1,time,s1) values('d1',2,2)", + "insert into sg1(tag1,time,s1) values('d1',604800001,2)", + "flush"); + for (String sql : sqls) { + RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + } + } + + public void testInsertTablet() { + List sqls = + Collections.singletonList( + "create table sg211 (tag1 string tag,t1 STRING ATTRIBUTE, s1 FLOAT field)"); + for (String sql : sqls) { + RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + } + String json = + "{\"database\":\"test\",\"column_catogories\":[\"TAG\",\"ATTRIBUTE\",\"FIELD\"],\"timestamps\":[1635232143960,1635232153960,1635232163960,1635232173960,1635232183960],\"column_names\":[\"tag1\",\"t1\",\"s1\"],\"data_types\":[\"STRING\",\"STRING\",\"FLOAT\"],\"values\":[[\"a11\",\"true\",11],[\"a11\",\"false\",22],[\"a13\",\"false1\",23],[\"a14\",\"false2\",24],[\"a15\",\"false3\",25]],\"table\":\"sg211\"}"; + rightInsertTablet(json); + } + + public void testInsertTabletNoDatabase() { + List sqls = + Collections.singletonList( + "create table sg211 (tag1 string tag,t1 STRING ATTRIBUTE, s1 FLOAT field)"); + for (String sql : sqls) { + RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + } + String json = + "{\"database\":\"\",\"column_catogories\":[\"TAG\",\"ATTRIBUTE\",\"FIELD\"],\"timestamps\":[1635232143960,1635232153960,1635232163960,1635232173960,1635232183960],\"column_names\":[\"tag1\",\"t1\",\"s1\"],\"data_types\":[\"STRING\",\"STRING\",\"FLOAT\"],\"values\":[[\"a11\",\"true\",11],[\"a11\",\"false\",22],[\"a13\",\"false1\",23],[\"a14\",\"false2\",24],[\"a15\",\"false3\",25]],\"table\":\"sg211\"}"; + JsonObject result = RestUtils.insertTablet(httpClient, port, json); + assertEquals(305, Integer.parseInt(result.get("code").toString())); + } + + public void testInsertTablet1() { + List sqls = + Collections.singletonList( + "create table sg211 (tag1 string tag,t1 STRING ATTRIBUTE, s1 FLOAT field)"); + for (String sql : sqls) { + RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + } + String json = + "{\"database\":\"test\",\"column_catogories\":[\"ATTRIBUTE\",\"FIELD\"],\"timestamps\":[1635232143960,1635232153960,1635232163960,1635232173960,1635232183960],\"column_names\":[\"id1\",\"t1\",\"s1\"],\"data_types\":[\"STRING\",\"STRING\",\"FLOAT\"],\"values\":[[\"a11\",\"true\",11],[\"a11\",\"false\",22],[\"a13\",\"false1\",23],[\"a14\",\"false2\",24],[\"a15\",\"false3\",25]],\"table\":\"sg211\"}"; + JsonObject result = RestUtils.insertTablet(httpClient, port, json); + assertEquals(305, Integer.parseInt(result.get("code").toString())); + assertEquals( + "column_names and column_catogories should have the same size,column_catogories and data_types should have the same size", + result.get("message").getAsString()); + } + + public void testInsertTablet2() { + List sqls = + Collections.singletonList( + "create table sg211 (tag1 string tag,t1 STRING ATTRIBUTE, s1 FLOAT field)"); + for (String sql : sqls) { + RestUtils.nonQuery(httpClient, port, sqlHandler("test", sql)); + } + String json = + "{\"database\":\"test\",\"column_catogories\":[\"TAG\",\"ATTRIBUTE\",\"FIELD\"],\"timestamps\":[1635232143960,1635232153960,1635232163960,1635232183960],\"column_names\":[\"tag1\",\"t1\",\"s1\"],\"data_types\":[\"STRING\",\"STRING\",\"FLOAT\"],\"values\":[[\"a11\",\"true\",11],[\"a11\",\"false\",22],[\"a13\",\"false1\",23],[\"a14\",\"false2\",24],[\"a15\",\"false3\",25]],\"table\":\"sg211\"}"; + JsonObject result = RestUtils.insertTablet(httpClient, port, json); + assertEquals(305, Integer.parseInt(result.get("code").toString())); + assertEquals( + "values and timestamps should have the same size", result.get("message").getAsString()); + } + + public void rightInsertTablet(String json) { + JsonObject result = RestUtils.insertTablet(httpClient, port, json); + assertEquals(200, Integer.parseInt(result.get("code").toString())); + JsonObject queryResult = + RestUtils.query( + httpClient, port, sqlHandler("test", "select tag1,t1,s1 from sg211 order by time")); + JsonArray jsonArray = queryResult.get("values").getAsJsonArray(); + JsonArray jsonArray1 = jsonArray.get(0).getAsJsonArray(); + assertEquals("a11", jsonArray1.get(0).getAsString()); + assertEquals("false", jsonArray1.get(1).getAsString()); + assertEquals(11f, jsonArray1.get(2).getAsFloat(), 0f); + } + + public void prepareTableData() { + for (int i = 0; i < sqls.length; i++) { + JsonObject jsonObject = new JsonObject(); + if (i > 0) { + jsonObject.addProperty("database", DATABASE); + } else { + jsonObject.addProperty("database", ""); + } + jsonObject.addProperty("sql", sqls[i]); + RestUtils.nonQuery(httpClient, port, jsonObject.toString()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java new file mode 100644 index 0000000000000..3abc382a87981 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.rest.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.*; +import org.apache.iotdb.itbase.env.BaseEnv; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBRestServiceInsertValuesIT { + + private int port = 18080; + private CloseableHttpClient httpClient = null; + + @Before + public void setUp() throws Exception { + BaseEnv baseEnv = EnvFactory.getEnv(); + baseEnv.getConfig().getDataNodeConfig().setEnableRestService(true); + baseEnv.initClusterEnvironment(); + DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + port = portConflictDataNodeWrapper.getRestServicePort(); + httpClient = HttpClientBuilder.create().build(); + } + + @After + public void tearDown() throws Exception { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static final String DATABASE = "test"; + + private static final String[] sqls = new String[] {"CREATE DATABASE t1"}; + + public void ping() { + HttpGet httpGet = new HttpGet("http://127.0.0.1:" + port + "/ping"); + CloseableHttpResponse response = null; + try { + for (int i = 0; i < 30; i++) { + try { + response = httpClient.execute(httpGet); + break; + } catch (Exception e) { + if (i == 29) { + throw e; + } + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + JsonObject result = JsonParser.parseString(message).getAsJsonObject(); + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(200, Integer.parseInt(result.get("code").toString())); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + } + + @Test + public void test() { + ping(); + prepareTableData(); + testInsertValues(); + testUpdatingValues(); + testInsertValuesWithSameTimestamp(); + testInsertWithWrongMeasurementNum1(); + testInsertWithWrongMeasurementNum2(); + testInsertWithDuplicatedMeasurements(); + testInsertMultiRows(); + testInsertLargeNumber(); + testExtendTextColumn(); + } + + public String sqlHandler(String database, String sql) { + JsonObject json = new JsonObject(); + json.addProperty("database", database); + json.addProperty("sql", sql); + return json.toString(); + } + + public void testInsertValues() { + List sqls = + Arrays.asList( + "create table wf01 (tag1 string tag, status boolean field, temperature float field)", + "insert into wf01(tag1, time, status, temperature) values ('wt01', 4000, true, 17.1)", + "insert into wf01(tag1, time, status, temperature) values ('wt01', 5000, true, 20.1)", + "insert into wf01(tag1, time, status, temperature) values ('wt01', 6000, true, 22)"); + for (String sql : sqls) { + nonQuery(sqlHandler("t1", sql)); + } + JsonObject jsonObject = query(sqlHandler("t1", "select time, status from wf01")); + JsonArray valuesList = jsonObject.getAsJsonArray("values"); + for (int i = 0; i < valuesList.size(); i++) { + JsonArray jsonArray = valuesList.get(i).getAsJsonArray(); + assertTrue(jsonArray.get(1).getAsBoolean()); + } + + jsonObject = query(sqlHandler("t1", "select time, status, temperature from wf01")); + valuesList = jsonObject.getAsJsonArray("values"); + for (int i = 0; i < valuesList.size(); i++) { + JsonArray jsonArray = valuesList.get(i).getAsJsonArray(); + if (i == 0) { + assertEquals(4000, jsonArray.get(0).getAsLong()); + assertTrue(jsonArray.get(1).getAsBoolean()); + assertEquals(17.1, jsonArray.get(2).getAsDouble(), 0.1); + } else if (i == 1) { + assertEquals(5000, jsonArray.get(0).getAsLong()); + assertTrue(jsonArray.get(1).getAsBoolean()); + assertEquals(20.1, jsonArray.get(2).getAsDouble(), 0.1); + } else if (i == 2) { + assertEquals(6000, jsonArray.get(0).getAsLong()); + assertTrue(jsonArray.get(1).getAsBoolean()); + assertEquals(22.0, jsonArray.get(2).getAsDouble(), 0.1); + } + } + } + + public void testUpdatingValues() { + List sqls = + Arrays.asList( + "create table wf03 (tag1 string tag, status boolean field, temperature float field)", + "insert into wf03(tag1, time, status, temperature) values ('wt01', 4000, true, 17.1)", + "insert into wf03(tag1, time, status) values ('wt01', 5000, true)", + "insert into wf03(tag1, time, temperature)values ('wt01', 5000, 20.1)", + "insert into wf03(tag1, time, temperature)values ('wt01', 6000, 22)"); + for (String sql : sqls) { + nonQuery(sqlHandler("t1", sql)); + } + + JsonObject jsonObject = query(sqlHandler("t1", "select time, status from wf03")); + JsonArray valuesList = jsonObject.getAsJsonArray("values"); + for (int i = 0; i < valuesList.size(); i++) { + JsonArray jsonArray = valuesList.get(i).getAsJsonArray(); + if (i >= 2) { + assertTrue(jsonArray.get(1).isJsonNull()); + } else { + assertTrue(jsonArray.get(1).getAsBoolean()); + } + } + jsonObject = query(sqlHandler("t1", "select time, status, temperature from wf03")); + valuesList = jsonObject.getAsJsonArray("values"); + for (int i = 0; i < valuesList.size(); i++) { + JsonArray jsonArray = valuesList.get(i).getAsJsonArray(); + if (i == 0) { + assertEquals(4000, jsonArray.get(0).getAsLong()); + assertTrue(jsonArray.get(1).getAsBoolean()); + assertEquals(17.1, jsonArray.get(2).getAsDouble(), 0.1); + } else if (i == 1) { + assertEquals(5000, jsonArray.get(0).getAsLong()); + assertTrue(jsonArray.get(1).getAsBoolean()); + assertEquals(20.1, jsonArray.get(2).getAsDouble(), 0.1); + } else if (i == 2) { + assertEquals(6000, jsonArray.get(0).getAsLong()); + assertTrue(jsonArray.get(1).isJsonNull()); + assertEquals(22.0f, jsonArray.get(2).getAsFloat(), 0.1); + } + } + } + + public void testInsertValuesWithSameTimestamp() { + List sqls = + Arrays.asList( + "create table sg3 (tag1 string tag, s2 double field, s1 double field)", + "insert into sg3(tag1,time,s2) values('d1',1,2)", + "insert into sg3(tag1,time,s1) values('d1',1,2)"); + for (String sql : sqls) { + nonQuery(sqlHandler("t1", sql)); + } + + JsonObject jsonObject = query(sqlHandler("t1", "select time, s1, s2 from sg3")); + JsonArray valuesList = jsonObject.getAsJsonArray("values"); + for (int i = 0; i < valuesList.size(); i++) { + JsonArray jsonArray = valuesList.get(i).getAsJsonArray(); + for (int c = 0; c < jsonArray.size(); c++) { + assertEquals(1, jsonArray.get(0).getAsLong()); + assertEquals(2.0d, jsonArray.get(1).getAsDouble(), 0.1); + assertEquals(2.0d, jsonArray.get(2).getAsDouble(), 0.1); + } + } + } + + public void testInsertWithWrongMeasurementNum1() { + nonQuery( + sqlHandler( + "t1", "create table wf04 (tag1 string tag, status int32, temperature int32 field)")); + JsonObject jsonObject = + nonQuery( + sqlHandler( + "t1", + "insert into wf04(tag1, time, status, temperature) values('wt01', 11000, 100)")); + assertEquals( + "701: Inconsistent numbers of non-time column names and values: 3-2", + jsonObject.get("code") + ": " + jsonObject.get("message").getAsString()); + } + + public void testInsertWithWrongMeasurementNum2() { + nonQuery( + sqlHandler( + "t1", "create table wf04 (tag1 string tag, status int32, temperature int32 field)")); + JsonObject jsonObject = + nonQuery( + sqlHandler( + "t1", + "insert into wf05(tag1, time, status, temperature) values('wt01', 11000, 100, 300, 400)")); + assertEquals( + "701: Inconsistent numbers of non-time column names and values: 3-4", + jsonObject.get("code") + ": " + jsonObject.get("message").getAsString()); + } + + public void testInsertWithDuplicatedMeasurements() { + nonQuery( + sqlHandler("t1", "create table wf07(tag1 string tag, s3 boolean field, status int32)")); + JsonObject jsonObject = + nonQuery( + sqlHandler( + "t1", + "insert into wf07(tag1, time, s3, status, status) values('wt01', 100, true, 20.1, 20.2)")); + assertEquals( + "701: Insertion contains duplicated measurement: status", + jsonObject.get("code") + ": " + jsonObject.get("message").getAsString()); + } + + public void testInsertMultiRows() { + nonQuery( + sqlHandler("t1", "create table sg8 (tag1 string tag, s1 int32 field, s2 int32 field)")); + JsonObject jsonObject = + nonQuery( + sqlHandler( + "t1", + "insert into sg8(tag1, time, s1, s2) values('d1', 10, 2, 2), ('d1', 11, 3, '3'), ('d1', 12,12.11,false)")); + assertEquals( + "507: Fail to insert measurements [s1, s2] caused by [data type is not consistent, input 12.11, registered INT32, data type is not consistent, input false, registered INT32]", + jsonObject.get("code") + ": " + jsonObject.get("message").getAsString()); + } + + public void testInsertLargeNumber() { + nonQuery( + sqlHandler("t1", "create table sg9 (tag1 string tag, s98 int64 field, s99 int64 field)")); + JsonObject jsonObject = + nonQuery( + sqlHandler( + "t1", + "insert into sg9(tag1, time, s98, s99) values('d1', 10, 2, 271840880000000000000000)")); + assertEquals( + "700: line 1:59: Invalid numeric literal: 271840880000000000000000", + jsonObject.get("code") + ": " + jsonObject.get("message").getAsString()); + } + + public void testExtendTextColumn() { + List sqls = + Arrays.asList( + "use t1", + "create table sg14 (tag1 string tag, s1 string field, s2 string field)", + "insert into sg14(tag1,time,s1,s2) values('d1',1,'test','test')", + "insert into sg14(tag1,time,s1,s2) values('d1',3,'test','test')", + "insert into sg14(tag1,time,s1,s2) values('d1',3,'test','test')", + "insert into sg14(tag1,time,s1,s2) values('d1',4,'test','test')", + "insert into sg14(tag1,time,s1,s3) values('d1',5,'test','test')", + "insert into sg14(tag1,time,s1,s2) values('d1',6,'test','test')", + "flush", + "insert into sg14(tag1,time,s1,s3) values('d1',7,'test','test')"); + try { + for (String sql : sqls) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("database", DATABASE); + jsonObject.addProperty("sql", sql); + nonQuery(jsonObject.toString()); + } + } catch (Exception ignore) { + + } + } + + public void prepareTableData() { + for (int i = 0; i < sqls.length; i++) { + JsonObject jsonObject = new JsonObject(); + if (i > 0) { + jsonObject.addProperty("database", DATABASE); + } else { + jsonObject.addProperty("database", ""); + } + jsonObject.addProperty("sql", sqls[i]); + nonQuery(jsonObject.toString()); + } + } + + public JsonObject query(String json) { + return RestUtils.query(httpClient, port, json); + } + + public JsonObject nonQuery(String json) { + return RestUtils.nonQuery(httpClient, port, json); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/RestUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/RestUtils.java new file mode 100644 index 0000000000000..cf70f9900f447 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/RestUtils.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.rest.it; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.junit.Assert.fail; + +public class RestUtils { + + public static JsonObject insertTablet(CloseableHttpClient httpClient, int port, String json) { + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:" + port + "/rest/table/v1/insertTablet"); + httpPost.setEntity(new StringEntity(json, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + return JsonParser.parseString(message).getAsJsonObject(); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + return new JsonObject(); + } + + private static HttpPost getHttpPost(String url) { + HttpPost httpPost = new HttpPost(url); + httpPost.addHeader("Content-type", "application/json; charset=utf-8"); + httpPost.setHeader("Accept", "application/json"); + String authorization = getAuthorization("root", "root"); + httpPost.setHeader("Authorization", authorization); + return httpPost; + } + + private static String getAuthorization(String username, String password) { + return Base64.getEncoder() + .encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)); + } + + public static JsonObject query(CloseableHttpClient httpClient, int port, String json) { + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:" + port + "/rest/table/v1/query"); + httpPost.setEntity(new StringEntity(json, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + return JsonParser.parseString(message).getAsJsonObject(); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + return new JsonObject(); + } + + public static JsonObject nonQuery(CloseableHttpClient httpClient, int port, String json) { + CloseableHttpResponse response = null; + try { + HttpPost httpPost = getHttpPost("http://127.0.0.1:" + port + "/rest/table/v1/nonQuery"); + httpPost.setEntity(new StringEntity(json, Charset.defaultCharset())); + response = httpClient.execute(httpPost); + HttpEntity responseEntity = response.getEntity(); + String message = EntityUtils.toString(responseEntity, "utf-8"); + return JsonParser.parseString(message).getAsJsonObject(); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + try { + if (response != null) { + response.close(); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + return new JsonObject(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java new file mode 100644 index 0000000000000..78bf4a1c574b0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -0,0 +1,832 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBDetailsColumnHeaders; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDatabaseIT { + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testManageDatabase() { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + + // create + statement.execute("create database test with (ttl='INF')"); + + // create duplicated database without IF NOT EXISTS + try { + statement.execute("create database test"); + fail("create database test shouldn't succeed because test already exists"); + } catch (final SQLException e) { + assertEquals("501: Database test already exists", e.getMessage()); + } + + // create duplicated database with IF NOT EXISTS + statement.execute("create database IF NOT EXISTS test"); + + // alter non-exist + try { + statement.execute("alter database test1 set properties ttl='INF'"); + fail("alter database test1 shouldn't succeed because test does not exist"); + } catch (final SQLException e) { + assertEquals("500: Database test1 doesn't exist", e.getMessage()); + } + + statement.execute("alter database if exists test1 set properties ttl='INF'"); + statement.execute("alter database test set properties ttl=default"); + + String[] databaseNames = new String[] {"test"}; + String[] TTLs = new String[] {"INF"}; + int[] schemaReplicaFactors = new int[] {1}; + int[] dataReplicaFactors = new int[] {1}; + int[] timePartitionInterval = new int[] {604800000}; + + // show + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + int cnt = 0; + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBColumnHeaders.size(); i++) { + assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + if (resultSet.getString(1).equals("information_schema")) { + continue; + } + assertEquals(databaseNames[cnt], resultSet.getString(1)); + assertEquals(TTLs[cnt], resultSet.getString(2)); + assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3)); + assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4)); + assertEquals(timePartitionInterval[cnt], resultSet.getLong(5)); + cnt++; + } + assertEquals(databaseNames.length, cnt); + } + + final int[] schemaRegionGroupNum = new int[] {0}; + final int[] dataRegionGroupNum = new int[] {0}; + // show + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { + int cnt = 0; + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBDetailsColumnHeaders.size(); i++) { + assertEquals( + showDBDetailsColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + if (resultSet.getString(1).equals("information_schema")) { + continue; + } + assertEquals(databaseNames[cnt], resultSet.getString(1)); + assertEquals(TTLs[cnt], resultSet.getString(2)); + assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3)); + assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4)); + assertEquals(timePartitionInterval[cnt], resultSet.getLong(5)); + assertEquals(schemaRegionGroupNum[cnt], resultSet.getInt(6)); + assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(7)); + cnt++; + } + assertEquals(databaseNames.length, cnt); + } + + // use + statement.execute("use test"); + + // use nonexistent database + try { + statement.execute("use test1"); + fail("use test1 shouldn't succeed because test1 doesn't exist"); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + // drop + statement.execute("drop database test"); + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + // Information_schema + assertTrue(resultSet.next()); + assertFalse(resultSet.next()); + } + + // drop nonexistent database without IF EXISTS + try { + statement.execute("drop database test"); + fail("drop database test shouldn't succeed because test doesn't exist"); + } catch (final SQLException e) { + assertEquals("500: Database test doesn't exist", e.getMessage()); + } + + // drop nonexistent database with IF EXISTS + statement.execute("drop database IF EXISTS test"); + + // Test create database with properties + statement.execute( + "create database test_prop with (ttl=300, schema_region_group_num=DEFAULT, time_partition_interval=100000)"); + databaseNames = new String[] {"test_prop"}; + TTLs = new String[] {"300"}; + timePartitionInterval = new int[] {100000}; + + // show + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + int cnt = 0; + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBColumnHeaders.size(); i++) { + assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + if (resultSet.getString(1).equals("information_schema")) { + continue; + } + assertEquals(databaseNames[cnt], resultSet.getString(1)); + assertEquals(TTLs[cnt], resultSet.getString(2)); + assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3)); + assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4)); + assertEquals(timePartitionInterval[cnt], resultSet.getLong(5)); + cnt++; + } + assertEquals(databaseNames.length, cnt); + } + + try { + statement.execute("create database test_prop_2 with (non_exist_prop=DEFAULT)"); + fail( + "create database test_prop_2 shouldn't succeed because the property key does not exist."); + } catch (final SQLException e) { + assertTrue( + e.getMessage(), + e.getMessage().contains("Unsupported database property key: non_exist_prop")); + } + + // create with strange name + try { + statement.execute("create database 1test"); + fail( + "create database 1test shouldn't succeed because 1test is not a legal identifier; identifiers must not start with a digit; surround the identifier with double quotes"); + } catch (final SQLException e) { + assertTrue(e.getMessage(), e.getMessage().contains("mismatched input '1'")); + } + + statement.execute("create database \"1test\""); + statement.execute("use \"1test\""); + statement.execute("drop database \"1test\""); + + try { + statement.execute("create database 1"); + fail("create database 1 shouldn't succeed because 1 is not a legal identifier"); + } catch (final SQLException e) { + assertTrue(e.getMessage(), e.getMessage().contains("mismatched input '1'")); + } + + statement.execute("create database \"1\""); + statement.execute("use \"1\""); + statement.execute("drop database \"1\""); + + try { + statement.execute("create database a.b"); + fail("create database a.b shouldn't succeed because a.b is not a legal identifier"); + } catch (final SQLException e) { + assertTrue(e.getMessage(), e.getMessage().contains("mismatched input '.'")); + } + + // Test length limitation + statement.execute( + "create database thisDatabaseLengthIsPreciselySixtyFourThusItCanBeNormallyCreated"); + + try { + statement.execute( + "create database thisDatabaseLengthHasExceededSixtyFourThusItCantBeNormallyCreated"); + fail( + "create database thisDatabaseLengthHasExceededSixtyFourThusItCantBeNormallyCreated shouldn't succeed because it's length has exceeded 64"); + } catch (final SQLException e) { + assertTrue( + e.getMessage(), + e.getMessage().contains("the length of database name shall not exceed 64")); + } + + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testDatabaseWithSpecificCharacters() throws SQLException { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + try { + statement.execute("create database \"````x.\""); + fail("create database ````x. shouldn't succeed because it contains '.'"); + } catch (final SQLException e) { + assertEquals( + "509: ````x. is not a legal path, because the database name can only contain english or chinese characters, numbers, backticks and underscores.", + e.getMessage()); + } + + try { + statement.execute("create database \"#\""); + fail("create database # shouldn't succeed because it contains illegal character '#'"); + } catch (final SQLException e) { + assertEquals( + "509: # is not a legal path, because the database name can only contain english or chinese characters, numbers, backticks and underscores.", + e.getMessage()); + } + + statement.execute("create database \"````x\""); + + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + assertTrue(resultSet.next()); + if (resultSet.getString(1).equals("information_schema")) { + assertTrue(resultSet.next()); + } + assertEquals("````x", resultSet.getString(1)); + assertFalse(resultSet.next()); + } + + statement.execute("use \"````x\""); + + statement.execute("create table table0 (a tag, b attribute, c int32)"); + + statement.execute("desc table0"); + statement.execute("desc \"````x\".table0"); + + statement.execute("show tables"); + statement.execute("show tables from \"````x\""); + + statement.execute("insert into table0 (time, a, b, c) values(0, '1', '2', 3)"); + statement.execute("insert into \"````x\".table0 (time, a, b, c) values(1, '1', '2', 3)"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("select a, b, c from \"````x\".table0 where time = 0"), + "a,b,c,", + Collections.singleton("1,2,3,")); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from table0"), + "a,b,", + Collections.singleton("1,2,")); + + statement.execute("update \"````x\".table0 set b = '4'"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from table0"), + "a,b,", + Collections.singleton("1,4,")); + } + } + + @Test + public void testInformationSchema() throws SQLException { + // Use a normal user to test visibility + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Test unsupported write plans + final Set writeSQLs = + new HashSet<>( + Arrays.asList( + "create database information_schema", + "drop database information_schema", + "create table information_schema.tableA ()", + "alter table information_schema.tableA add column a id", + "alter table information_schema.tableA set properties ttl=default", + "insert into information_schema.tables (database) values('db')", + "update information_schema.tables set status='RUNNING'")); + + for (final String writeSQL : writeSQLs) { + try { + statement.execute(writeSQL); + fail("information_schema does not support write"); + } catch (final SQLException e) { + assertEquals( + "701: The database 'information_schema' can only be queried", e.getMessage()); + } + } + + statement.execute("use information_schema"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show databases"), + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("show tables"), + "TableName,TTL(ms),", + new HashSet<>( + Arrays.asList( + "databases,INF,", + "tables,INF,", + "columns,INF,", + "queries,INF,", + "regions,INF,", + "topics,INF,", + "pipe_plugins,INF,", + "pipes,INF,", + "subscriptions,INF,", + "views,INF,", + "models,INF,", + "functions,INF,", + "configurations,INF,", + "keywords,INF,"))); + + TestUtils.assertResultSetEqual( + statement.executeQuery("desc databases"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "database,STRING,TAG,", + "ttl(ms),STRING,ATTRIBUTE,", + "schema_replication_factor,INT32,ATTRIBUTE,", + "data_replication_factor,INT32,ATTRIBUTE,", + "time_partition_interval,INT64,ATTRIBUTE,", + "schema_region_group_num,INT32,ATTRIBUTE,", + "data_region_group_num,INT32,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc tables"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "database,STRING,TAG,", + "table_name,STRING,TAG,", + "ttl(ms),STRING,ATTRIBUTE,", + "status,STRING,ATTRIBUTE,", + "comment,STRING,ATTRIBUTE,", + "table_type,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc columns"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "database,STRING,TAG,", + "table_name,STRING,TAG,", + "column_name,STRING,TAG,", + "datatype,STRING,ATTRIBUTE,", + "category,STRING,ATTRIBUTE,", + "status,STRING,ATTRIBUTE,", + "comment,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc queries"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "query_id,STRING,TAG,", + "start_time,TIMESTAMP,ATTRIBUTE,", + "datanode_id,INT32,ATTRIBUTE,", + "elapsed_time,FLOAT,ATTRIBUTE,", + "statement,STRING,ATTRIBUTE,", + "user,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc pipes"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "id,STRING,TAG,", + "creation_time,TIMESTAMP,ATTRIBUTE,", + "state,STRING,ATTRIBUTE,", + "pipe_source,STRING,ATTRIBUTE,", + "pipe_processor,STRING,ATTRIBUTE,", + "pipe_sink,STRING,ATTRIBUTE,", + "exception_message,STRING,ATTRIBUTE,", + "remaining_event_count,INT64,ATTRIBUTE,", + "estimated_remaining_seconds,DOUBLE,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc pipe_plugins"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "plugin_name,STRING,TAG,", + "plugin_type,STRING,ATTRIBUTE,", + "class_name,STRING,ATTRIBUTE,", + "plugin_jar,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc topics"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList("topic_name,STRING,TAG,", "topic_configs,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc subscriptions"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "topic_name,STRING,TAG,", + "consumer_group_name,STRING,TAG,", + "subscribed_consumers,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc views"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "database,STRING,TAG,", + "table_name,STRING,TAG,", + "view_definition,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc models"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "model_id,STRING,TAG,", + "model_type,STRING,ATTRIBUTE,", + "state,STRING,ATTRIBUTE,", + "configs,STRING,ATTRIBUTE,", + "notes,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc functions"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "function_table,STRING,TAG,", + "function_type,STRING,ATTRIBUTE,", + "class_name(udf),STRING,ATTRIBUTE,", + "state,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc configurations"), + "ColumnName,DataType,Category,", + new HashSet<>(Arrays.asList("variable,STRING,TAG,", "value,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc keywords"), + "ColumnName,DataType,Category,", + new HashSet<>(Arrays.asList("word,STRING,TAG,", "reserved,INT32,ATTRIBUTE,"))); + + // Only root user is allowed + Assert.assertThrows(SQLException.class, () -> statement.execute("select * from regions")); + Assert.assertThrows(SQLException.class, () -> statement.execute("select * from pipes")); + Assert.assertThrows(SQLException.class, () -> statement.execute("select * from topics")); + Assert.assertThrows( + SQLException.class, () -> statement.execute("select * from subscriptions")); + Assert.assertThrows( + SQLException.class, () -> statement.execute("select * from configurations")); + + // No auth needed + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select * from pipe_plugins where plugin_name = 'IOTDB-THRIFT-SINK'"), + "plugin_name,plugin_type,class_name,plugin_jar,", + Collections.singleton( + "IOTDB-THRIFT-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.thrift.IoTDBThriftConnector,null,")); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select model_id from information_schema.models where model_type = 'BUILT_IN_FORECAST'"), + "model_id,", + new HashSet<>( + Arrays.asList( + "_timerxl,", + "_STLForecaster,", + "_NaiveForecaster,", + "_ARIMA,", + "_ExponentialSmoothing,"))); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select distinct(function_type) from information_schema.functions"), + "function_type,", + new HashSet<>( + Arrays.asList( + "built-in scalar function,", + "built-in aggregate function,", + "built-in table function,"))); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select * from information_schema.keywords where reserved > 0 limit 1"), + "word,reserved,", + Collections.singleton("AINODES,1,")); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Test table query + statement.execute("use information_schema"); + + statement.execute("create database test"); + statement.execute( + "create table test.test (a tag, b attribute, c int32 comment 'turbine') comment 'test'"); + statement.execute( + "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100) AS root.a.**"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("select * from databases"), + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,", + new HashSet<>( + Arrays.asList( + "information_schema,INF,null,null,null,null,null,", + "test,INF,1,1,604800000,0,0,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from tables where status = 'USING'"), + "database,table_name,ttl(ms),status,comment,table_type,", + new HashSet<>( + Arrays.asList( + "information_schema,databases,INF,USING,null,SYSTEM VIEW,", + "information_schema,tables,INF,USING,null,SYSTEM VIEW,", + "information_schema,columns,INF,USING,null,SYSTEM VIEW,", + "information_schema,queries,INF,USING,null,SYSTEM VIEW,", + "information_schema,regions,INF,USING,null,SYSTEM VIEW,", + "information_schema,topics,INF,USING,null,SYSTEM VIEW,", + "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,", + "information_schema,pipes,INF,USING,null,SYSTEM VIEW,", + "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,", + "information_schema,views,INF,USING,null,SYSTEM VIEW,", + "information_schema,models,INF,USING,null,SYSTEM VIEW,", + "information_schema,functions,INF,USING,null,SYSTEM VIEW,", + "information_schema,configurations,INF,USING,null,SYSTEM VIEW,", + "information_schema,keywords,INF,USING,null,SYSTEM VIEW,", + "test,test,INF,USING,test,BASE TABLE,", + "test,view_table,100,USING,null,VIEW FROM TREE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from tables where status = 'USING'"), + "count(devices),", + Collections.singleton("16,")); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select * from columns where table_name = 'queries' or database = 'test'"), + "database,table_name,column_name,datatype,category,status,comment,", + new HashSet<>( + Arrays.asList( + "information_schema,queries,query_id,STRING,TAG,USING,null,", + "information_schema,queries,start_time,TIMESTAMP,ATTRIBUTE,USING,null,", + "information_schema,queries,datanode_id,INT32,ATTRIBUTE,USING,null,", + "information_schema,queries,elapsed_time,FLOAT,ATTRIBUTE,USING,null,", + "information_schema,queries,statement,STRING,ATTRIBUTE,USING,null,", + "information_schema,queries,user,STRING,ATTRIBUTE,USING,null,", + "test,test,time,TIMESTAMP,TIME,USING,null,", + "test,test,a,STRING,TAG,USING,null,", + "test,test,b,STRING,ATTRIBUTE,USING,null,", + "test,test,c,INT32,FIELD,USING,turbine,", + "test,view_table,time,TIMESTAMP,TIME,USING,null,", + "test,view_table,tag1,STRING,TAG,USING,null,", + "test,view_table,tag2,STRING,TAG,USING,null,", + "test,view_table,s11,INT32,FIELD,USING,null,", + "test,view_table,s3,INT32,FIELD,USING,null,"))); + + statement.execute( + "create pipe a2b with source('double-living'='true') with sink ('sink'='write-back-sink')"); + TestUtils.assertResultSetEqual( + statement.executeQuery("select id from pipes where creation_time > 0"), + "id,", + Collections.singleton("a2b,")); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select * from pipe_plugins where plugin_name = 'IOTDB-THRIFT-SINK'"), + "plugin_name,plugin_type,class_name,plugin_jar,", + Collections.singleton( + "IOTDB-THRIFT-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.thrift.IoTDBThriftConnector,null,")); + + statement.execute("create topic tp with ('start-time'='2025-01-13T10:03:19.229+08:00')"); + TestUtils.assertResultSetEqual( + statement.executeQuery("select * from topics where topic_name = 'tp'"), + "topic_name,topic_configs,", + Collections.singleton( + "tp,{__system.sql-dialect=table, start-time=2025-01-13T10:03:19.229+08:00},")); + + TestUtils.assertResultSetEqual( + statement.executeQuery("select * from views"), + "database,table_name,view_definition,", + Collections.singleton( + "test,view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.a.**,")); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select model_id from information_schema.models where model_type = 'BUILT_IN_FORECAST'"), + "model_id,", + new HashSet<>( + Arrays.asList( + "_timerxl,", + "_STLForecaster,", + "_NaiveForecaster,", + "_ARIMA,", + "_ExponentialSmoothing,"))); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select distinct(function_type) from information_schema.functions"), + "function_type,", + new HashSet<>( + Arrays.asList( + "built-in scalar function,", + "built-in aggregate function,", + "built-in table function,"))); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select value from information_schema.configurations where variable = 'TimestampPrecision'"), + "value,", + Collections.singleton("ms,")); + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select * from information_schema.keywords where reserved > 0 limit 1"), + "word,reserved,", + Collections.singleton("AINODES,1,")); + } + } + + @Test + public void testMixedDatabase() throws SQLException { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database test"); + statement.execute("use test"); + statement.execute("create table table1(id1 tag, s1 string)"); + statement.execute("insert into table1 values(0, 'd1', null), (1,'d1', 1)"); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database root.test"); + statement.execute( + "alter database root.test WITH SCHEMA_REGION_GROUP_NUM=2, DATA_REGION_GROUP_NUM=3"); + statement.execute("insert into root.test.d1 (s1) values(1)"); + statement.execute("drop database root.test"); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { + assertTrue(resultSet.next()); + if (resultSet.getString(1).equals("information_schema")) { + assertTrue(resultSet.next()); + } + assertEquals("test", resultSet.getString(1)); + assertFalse(resultSet.next()); + } + + // Test adjustMaxRegionGroupNum + statement.execute("use test"); + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD)"); + statement.execute( + "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); + + statement.execute("create database test1"); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database root.test"); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("drop database test"); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + TestUtils.assertResultSetSize(statement.executeQuery("show databases"), 1); + } + } + + @Test + public void testDBAuth() throws SQLException { + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + adminStmt.execute("create database db"); + adminStmt.execute( + "create pipe a2b with source('double-living'='true') with sink ('sink'='write-back-sink')"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + TestUtils.assertResultSetEqual( + userStmt.executeQuery("show databases"), + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,")); + TestUtils.assertResultSetEqual( + userStmt.executeQuery("select * from information_schema.databases"), + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,", + Collections.singleton("information_schema,INF,null,null,null,null,null,")); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT SELECT ON DATABASE DB to user test"); + + // Information_schema does not support grant & revoke + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("GRANT SELECT ON DATABASE information_schema to user test"); + }); + + Assert.assertThrows( + SQLException.class, + () -> { + adminStmt.execute("REVOKE SELECT ON information_schema.tables from user test"); + }); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + try (final ResultSet resultSet = userStmt.executeQuery("SHOW DATABASES")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBColumnHeaders.size(); i++) { + assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + Assert.assertTrue(resultSet.next()); + if (resultSet.getString(1).equals("information_schema")) { + assertTrue(resultSet.next()); + } + assertEquals("db", resultSet.getString(1)); + Assert.assertFalse(resultSet.next()); + } + + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("alter database db set properties ttl=6600000"); + }); + + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("drop database db"); + }); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT DROP ON ANY to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + userStmt.execute("drop database db"); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java new file mode 100644 index 0000000000000..7b023b4e9f6a7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBDeviceIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultSchemaRegionGroupNumPerDatabase(2); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDevice() throws SQLException { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Prepare data + statement.execute("create database test"); + statement.execute("use test"); + statement.execute( + "create table table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD)"); + statement.execute( + "create table table0(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD)"); + statement.execute( + "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('1', '木兰', '3', 'A', 37.6, 111.1)"); + + // Test plain show / count + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from table0"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,A,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from table0"), + "count(devices),", + Collections.singleton("1,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from table1"), + "region_id,plant_id,device_id,model,", + Collections.emptySet()); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from table1"), + "count(devices),", + Collections.singleton("0,")); + + // Test show / count with where expression + // Test AND + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where region_id between '0' and '2' and model = 'A'"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,A,")); + // Test OR + TestUtils.assertResultSetEqual( + statement.executeQuery( + "count devices from table0 where region_id = '1' or plant_id like '%木兰'"), + "count(devices),", + Collections.singleton("1,")); + // Test complicated query + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where region_id < plant_id offset 0 limit 1"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,A,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from table0 where region_id < plant_id limit 0"), + "region_id,plant_id,device_id,model,", + Collections.emptySet()); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from table0 where region_id < plant_id"), + "count(devices),", + Collections.singleton("1,")); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "count devices from table0 where substring(region_id, cast((cast(device_id as int32) - 2) as int32), 1) < plant_id"), + "count(devices),", + Collections.singleton("1,")); + // Test get from cache + statement.executeQuery( + "select * from table0 where region_id = '1' and plant_id in ('木兰', '5') and device_id = '3'"); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where region_id = '1' and plant_id in ('3', '木兰') and device_id = '3' offset 0 limit ALL"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,A,")); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where region_id = '1' and plant_id in ('木兰', '5') and device_id = '3' offset 100000000000 limit 100000000000"), + "region_id,plant_id,device_id,model,", + Collections.emptySet()); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "count devices from table0 where region_id = '1' and plant_id in ('3', '木兰') and device_id = '3'"), + "count(devices),", + Collections.singleton("1,")); + // Test filter + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from table0 where region_id >= '2'"), + "count(devices),", + Collections.singleton("0,")); + // Test cache with complicated filter + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where region_id = '1' and plant_id in ('木兰', '5') and device_id = '3' and device_id between region_id and plant_id"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,A,")); + + try { + statement.executeQuery("show devices from table2"); + fail("Show devices shall fail for non-exist table"); + } catch (final Exception e) { + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); + } + + try { + statement.executeQuery("count devices from table2"); + fail("Count devices shall fail for non-exist table"); + } catch (final Exception e) { + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); + } + + try { + statement.executeQuery("show devices from table0 where temperature = 37.6"); + fail("Show devices shall fail for measurement predicate"); + } catch (final Exception e) { + assertEquals( + "701: The TIME/FIELD columns are currently not allowed in devices related operations", + e.getMessage()); + } + + try { + statement.executeQuery("count devices from table0 where a = 1"); + fail("Count devices shall fail for non-exist column"); + } catch (final Exception e) { + assertEquals("616: Column 'a' cannot be resolved", e.getMessage()); + } + + // Test fully qualified name + statement.execute("create database test2"); + statement.execute("use test2"); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from test.table0"), + "count(devices),", + Collections.singleton("1,")); + + // Test update + statement.execute("use test"); + try { + statement.execute("update table2 set model = '1'"); + fail("Update shall fail for non-exist table"); + } catch (final Exception e) { + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); + } + + try { + statement.execute("update table0 set device_id = '1'"); + fail("Update shall fail for tag"); + } catch (final Exception e) { + assertEquals("701: Update can only specify attribute columns.", e.getMessage()); + } + + try { + statement.execute("update table0 set model = '1', model = '2'"); + fail("Update shall fail if an attribute occurs twice"); + } catch (final Exception e) { + assertEquals("701: Update attribute shall specify a attribute only once.", e.getMessage()); + } + + try { + statement.execute("update table0 set col = '1'"); + fail("Update shall fail for non-exist column"); + } catch (final Exception e) { + assertEquals("616: Column 'col' cannot be resolved", e.getMessage()); + } + + try { + statement.execute("update table0 set model = cast(device_id as int32)"); + fail("Update shall fail when result type mismatch"); + } catch (final Exception e) { + assertEquals( + "507: Result type mismatch for attribute 'model', expected class org.apache.tsfile.utils.Binary, actual class java.lang.Integer", + e.getMessage()); + } + + // Test filter with no effect + statement.execute("update table0 set model = null where model = 'A' and model = 'B'"); + + // Test null + statement.execute("update table0 set model = null where model <> substring(device_id, 1, 1)"); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where substring(region_id, 1, 1) in ('1', '2') and 1 + 1 = 2"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,null,")); + + // Test common result column + statement.execute( + "update table0 set model = substring(device_id, 1, 1) where cast(region_id as int32) + cast(device_id as int32) = 4 and region_id = '1'"); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from table0 where substring(region_id, 1, 1) in ('1', '2') and 1 + 1 = 2"), + "region_id,plant_id,device_id,model,", + Collections.singleton("1,木兰,3,3,")); + + // Test limit / offset from multi regions + statement.execute( + "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('2', '5', '3', 'A', 37.6, 111.1)"); + TestUtils.assertResultSetSize( + statement.executeQuery("show devices from table0 offset 1 limit 1"), 1); + + // Test delete devices with no effect + statement.execute("delete devices from table0 where region_id = '1' and region_id = '2'"); + + // Test delete devices + statement.execute("delete devices from table0 where region_id = '1' and plant_id = '木兰'"); + TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 1); + + // Test successfully Invalidate cache + statement.execute( + "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('1', '木兰', '3', 'A', 37.6, 111.1)"); + TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 2); + + // Test successfully delete data + TestUtils.assertResultSetSize( + statement.executeQuery("select * from table0 where region_id = '1'"), 1); + + try { + statement.executeQuery("delete devices from table0 where time = 1"); + fail("Delete devices shall fail when specifies non tag column"); + } catch (final Exception e) { + assertEquals( + "701: The TIME/FIELD columns are currently not allowed in devices related operations", + e.getMessage()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java new file mode 100644 index 0000000000000..2e4572d969dfc --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -0,0 +1,1049 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableDetailsColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesDetailsColumnHeaders; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testManageTable() { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + + statement.execute("create database test1"); + statement.execute("create database test2 with (ttl=3000000)"); + + // should specify database before create table + try { + statement.execute( + "create table table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD)"); + fail(); + } catch (final SQLException e) { + assertEquals("701: database is not specified", e.getMessage()); + } + + // Show tables shall succeed in a newly created database with no tables + try (final ResultSet resultSet = statement.executeQuery("SHOW tables from test1")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showTablesColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + assertFalse(resultSet.next()); + } + + // or use full qualified table name + // test "TTL=INF" + // "FIELD" can be omitted when type is specified + // "STRING" can be omitted when tag/attribute is specified + statement.execute( + "create table test1.table1(time TIMESTAMP TIME COMMENT 'column_comment', region_id STRING TAG, plant_id STRING TAG, device_id TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE) comment 'test' with (TTL='INF')"); + + try { + statement.execute( + "create table test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD)"); + fail(); + } catch (final SQLException e) { + assertEquals("551: Table 'test1.table1' already exists.", e.getMessage()); + } + + String[] tableNames = new String[] {"table1"}; + String[] ttls = new String[] {"INF"}; + String[] statuses = new String[] {"USING"}; + String[] comments = new String[] {"test"}; + + statement.execute("use test2"); + + // show tables by specifying another database + // Check duplicate create table won't affect table state + // using SHOW tables in + try (final ResultSet resultSet = statement.executeQuery("SHOW tables details in test1")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showTablesDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showTablesDetailsColumnHeaders.size(); i++) { + assertEquals( + showTablesDetailsColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(tableNames[cnt], resultSet.getString(1)); + assertEquals(ttls[cnt], resultSet.getString(2)); + assertEquals(statuses[cnt], resultSet.getString(3)); + assertEquals(comments[cnt], resultSet.getString(4)); + cnt++; + } + assertEquals(tableNames.length, cnt); + } + + // Alter table properties + statement.execute("alter table test1.table1 set properties ttl=1000000"); + ttls = new String[] {"1000000"}; + + // Alter non-exist table + try { + statement.execute("alter table test1.nonExist set properties ttl=1000000"); + } catch (final SQLException e) { + assertEquals("550: Table 'test1.nonexist' does not exist", e.getMessage()); + } + + // If exists + statement.execute("alter table if exists test1.nonExist set properties ttl=1000000"); + + // Alter non-supported properties + try { + statement.execute("alter table test1.table1 set properties nonSupport=1000000"); + } catch (final SQLException e) { + assertEquals("701: Table property 'nonsupport' is currently not allowed.", e.getMessage()); + } + + statement.execute("comment on table test1.table1 is 'new_test'"); + comments = new String[] {"new_test"}; + // using SHOW tables from + try (final ResultSet resultSet = statement.executeQuery("SHOW tables details from test1")) { + int cnt = 0; + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showTablesDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showTablesDetailsColumnHeaders.size(); i++) { + assertEquals( + showTablesDetailsColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(tableNames[cnt], resultSet.getString(1)); + assertEquals(ttls[cnt], resultSet.getString(2)); + assertEquals(comments[cnt], resultSet.getString(4)); + cnt++; + } + assertEquals(tableNames.length, cnt); + } + + // Set back to default + statement.execute("alter table test1.table1 set properties ttl=DEFAULT"); + ttls = new String[] {"INF"}; + + try (final ResultSet resultSet = statement.executeQuery("SHOW tables from test1")) { + int cnt = 0; + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showTablesColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(tableNames[cnt], resultSet.getString(1)); + assertEquals(ttls[cnt], resultSet.getString(2)); + cnt++; + } + assertEquals(tableNames.length, cnt); + } + + // Create if not exist + statement.execute( + "create table if not exists test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD)"); + + try { + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (UNKNOWN=3600000)"); + fail(); + } catch (final SQLException e) { + assertEquals("701: Table property 'unknown' is currently not allowed.", e.getMessage()); + } + + try { + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=null)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: ttl value must be a LongLiteral, but now is NullLiteral, value: null", + e.getMessage()); + } + + try { + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=-1)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: ttl value must be equal to or greater than 0, but now is: -1", e.getMessage()); + } + + try { + statement.execute( + "create table table2(region_id TEXT TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: DataType of TAG Column should only be STRING, current is TEXT", e.getMessage()); + } + + try { + statement.execute( + "create table table2(region_id INT32 TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: DataType of TAG Column should only be STRING, current is INT32", e.getMessage()); + } + + try { + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model TEXT ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: DataType of ATTRIBUTE Column should only be STRING, current is TEXT", + e.getMessage()); + } + + try { + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model DOUBLE ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: DataType of ATTRIBUTE Column should only be STRING, current is DOUBLE", + e.getMessage()); + } + + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD) with (TTL=6600000)"); + + statement.execute("alter table table2 add column speed DOUBLE FIELD COMMENT 'fast'"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create table table2"), + "Table,Create Table,", + Collections.singleton( + "table2,CREATE TABLE \"table2\" (\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000),")); + + try { + statement.execute("alter table table2 add column speed DOUBLE FIELD"); + } catch (final SQLException e) { + assertEquals("552: Column 'speed' already exist", e.getMessage()); + } + + statement.execute("alter table table2 add column if not exists speed DOUBLE FIELD"); + + try { + statement.execute("alter table table3 add column speed DOUBLE FIELD"); + } catch (final SQLException e) { + assertEquals("550: Table 'test2.table3' does not exist", e.getMessage()); + } + + statement.execute("alter table if exists table3 add column speed DOUBLE FIELD"); + + // Test create table with only time column + statement.execute("create table table3()"); + + tableNames = new String[] {"table3", "table2"}; + ttls = new String[] {"3000000", "6600000"}; + + // show tables from current database + try (final ResultSet resultSet = statement.executeQuery("SHOW tables")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showTablesColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(tableNames[cnt], resultSet.getString(1)); + assertEquals(ttls[cnt], resultSet.getString(2)); + cnt++; + } + assertEquals(tableNames.length, cnt); + } + + // Will not affect the manual "6600000" + statement.execute("alter database test2 set properties ttl=6600000"); + statement.execute("alter database test2 set properties ttl=DEFAULT"); + + statement.execute("alter table table3 set properties ttl=1000000"); + statement.execute("alter table table3 set properties ttl=DEFAULT"); + + ttls = new String[] {"INF", "6600000"}; + // The table3's ttl shall be "INF" + try (final ResultSet resultSet = statement.executeQuery("SHOW tables")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showTablesColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(tableNames[cnt], resultSet.getString(1)); + assertEquals(ttls[cnt], resultSet.getString(2)); + cnt++; + } + assertEquals(tableNames.length, cnt); + } + + // show tables from a non-exist database + try { + statement.executeQuery("SHOW tables from test3"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test3", e.getMessage()); + } + + // describe + try { + statement.executeQuery("describe table1"); + fail(); + } catch (final SQLException e) { + assertEquals("550: Table 'test2.table1' does not exist.", e.getMessage()); + } + + String[] columnNames = + new String[] { + "time", "region_id", "plant_id", "device_id", "model", "temperature", "humidity" + }; + String[] dataTypes = + new String[] {"TIMESTAMP", "STRING", "STRING", "STRING", "STRING", "FLOAT", "DOUBLE"}; + String[] categories = + new String[] {"TIME", "TAG", "TAG", "TAG", "ATTRIBUTE", "FIELD", "FIELD"}; + + try (final ResultSet resultSet = statement.executeQuery("describe test1.table1")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(describeTableColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < describeTableColumnHeaders.size(); i++) { + assertEquals( + describeTableColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(columnNames[cnt], resultSet.getString(1)); + assertEquals(dataTypes[cnt], resultSet.getString(2)); + assertEquals(categories[cnt], resultSet.getString(3)); + cnt++; + } + assertEquals(columnNames.length, cnt); + } + + columnNames = new String[] {"time", "region_id", "plant_id", "color", "temperature", "speed"}; + dataTypes = new String[] {"TIMESTAMP", "STRING", "STRING", "STRING", "FLOAT", "DOUBLE"}; + categories = new String[] {"TIME", "TAG", "TAG", "ATTRIBUTE", "FIELD", "FIELD"}; + + try (final ResultSet resultSet = statement.executeQuery("desc table2")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(describeTableColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < describeTableColumnHeaders.size(); i++) { + assertEquals( + describeTableColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(columnNames[cnt], resultSet.getString(1)); + assertEquals(dataTypes[cnt], resultSet.getString(2)); + assertEquals(categories[cnt], resultSet.getString(3)); + cnt++; + } + assertEquals(columnNames.length, cnt); + } + + statement.execute( + "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); + + // Test drop column + statement.execute("alter table table2 drop column color"); + + // Test comment + // Before + columnNames = new String[] {"time", "region_id", "plant_id", "temperature", "speed"}; + dataTypes = new String[] {"TIMESTAMP", "STRING", "STRING", "FLOAT", "DOUBLE"}; + categories = new String[] {"TIME", "TAG", "TAG", "FIELD", "FIELD"}; + statuses = new String[] {"USING", "USING", "USING", "USING", "USING"}; + + comments = new String[] {null, null, null, null, "fast"}; + try (final ResultSet resultSet = statement.executeQuery("describe table2 details")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(describeTableDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < describeTableDetailsColumnHeaders.size(); i++) { + assertEquals( + describeTableDetailsColumnHeaders.get(i).getColumnName(), + metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(columnNames[cnt], resultSet.getString(1)); + assertEquals(dataTypes[cnt], resultSet.getString(2)); + assertEquals(categories[cnt], resultSet.getString(3)); + assertEquals(statuses[cnt], resultSet.getString(4)); + assertEquals(comments[cnt], resultSet.getString(5)); + cnt++; + } + assertEquals(columnNames.length, cnt); + } + + // After + statement.execute("COMMENT ON COLUMN table2.region_id IS '重庆'"); + statement.execute("COMMENT ON COLUMN table2.region_id IS NULL"); + statement.execute("COMMENT ON COLUMN test2.table2.time IS 'recent'"); + statement.execute("COMMENT ON COLUMN test2.table2.region_id IS ''"); + + comments = new String[] {"recent", "", null, null, "fast"}; + try (final ResultSet resultSet = statement.executeQuery("describe table2 details")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(describeTableDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < describeTableDetailsColumnHeaders.size(); i++) { + assertEquals( + describeTableDetailsColumnHeaders.get(i).getColumnName(), + metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(columnNames[cnt], resultSet.getString(1)); + assertEquals(dataTypes[cnt], resultSet.getString(2)); + assertEquals(categories[cnt], resultSet.getString(3)); + assertEquals(statuses[cnt], resultSet.getString(4)); + assertEquals(comments[cnt], resultSet.getString(5)); + cnt++; + } + assertEquals(columnNames.length, cnt); + } + + statement.execute("alter table table2 drop column speed"); + + try { + statement.executeQuery("select color from table2"); + fail(); + } catch (final SQLException e) { + assertEquals("616: Column 'color' cannot be resolved", e.getMessage()); + } + + try { + statement.executeQuery("select speed from table2"); + fail(); + } catch (final SQLException e) { + assertEquals("616: Column 'speed' cannot be resolved", e.getMessage()); + } + + try { + statement.execute("alter table table2 drop column speed"); + } catch (final SQLException e) { + assertEquals("616: Column speed in table 'test2.table2' does not exist.", e.getMessage()); + } + + try { + statement.execute("alter table table2 drop column time"); + } catch (final SQLException e) { + assertEquals("701: Dropping tag or time column is not supported.", e.getMessage()); + } + + // test data deletion by drop column + statement.execute("alter table table2 add column speed double"); + TestUtils.assertResultSetEqual( + statement.executeQuery("select speed from table2"), + "speed,", + Collections.singleton("null,")); + + statement.execute("drop table table2"); + try { + statement.executeQuery("describe table2"); + fail(); + } catch (final SQLException e) { + assertEquals("550: Table 'test2.table2' does not exist.", e.getMessage()); + } + statement.execute( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from table2"), + "count(devices),", + Collections.singleton("0,")); + + // Test data deletion by drop table + statement.execute( + "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); + TestUtils.assertResultSetSize(statement.executeQuery("select * from table2"), 1); + + try { + statement.executeQuery("describe test3.table3"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test3", e.getMessage()); + } + + statement.execute("drop database test1"); + + // Test error messages + try { + statement.executeQuery("SHOW tables from test1"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + try { + statement.execute("create table test1.test()"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + try { + statement.execute("alter table test1.test add column a int32"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + try { + statement.execute("alter table test1.test drop column a"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + try { + statement.execute("alter table test1.test set properties ttl=default"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + try { + statement.execute("desc test1.test"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + try { + statement.execute("drop table test1.test"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); + } + + // Test time column + statement.execute("create table test100 (time time)"); + statement.execute("create table test101 (time timestamp time)"); + + try { + statement.execute("create table test102 (time timestamp tag)"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: The time column category shall be bounded with column name 'time'.", + e.getMessage()); + } + + try { + statement.execute("create table test102 (time tag)"); + fail(); + } catch (final SQLException e) { + assertEquals("701: The time column's type shall be 'timestamp'.", e.getMessage()); + } + + try { + statement.execute("create table test102 (time time, time time)"); + fail(); + } catch (final SQLException e) { + assertEquals("701: Columns in table shall not share the same name time.", e.getMessage()); + } + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testTableAuth() throws SQLException { + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + adminStmt.execute("create database db"); + adminStmt.execute("use db"); + adminStmt.execute("create table test (a tag, b attribute, c int32)"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + Assert.assertThrows(SQLException.class, () -> userStmt.execute("select * from db.test")); + TestUtils.assertResultSetEqual( + userStmt.executeQuery("select * from information_schema.tables where database = 'db'"), + "database,table_name,ttl(ms),status,comment,table_type,", + Collections.emptySet()); + TestUtils.assertResultSetEqual( + userStmt.executeQuery("select * from information_schema.columns where database = 'db'"), + "database,table_name,column_name,datatype,category,status,comment,", + Collections.emptySet()); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT SELECT ON db.test to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + try (final ResultSet resultSet = userStmt.executeQuery("SHOW DATABASES")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBColumnHeaders.size(); i++) { + assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + Assert.assertTrue(resultSet.next()); + if (resultSet.getString(1).equals("information_schema")) { + assertTrue(resultSet.next()); + } + assertEquals("db", resultSet.getString(1)); + Assert.assertFalse(resultSet.next()); + } + + userStmt.execute("select * from db.test"); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT DROP ON DATABASE DB to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + userStmt.execute("use db"); + userStmt.execute("drop table test"); + } + } + + // Test deadlock + @Test(timeout = 60000) + public void testConcurrentAutoCreateAndDropColumn() throws Exception { + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection(); + final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create database db1"); + session.executeNonQueryStatement("USE \"db1\""); + + final StringBuilder sb = new StringBuilder("CREATE TABLE table8 (tag1 string tag"); + for (int i = 0; i < 100; ++i) { + sb.append(String.format(", m%s string", i)); + } + sb.append(")"); + session.executeNonQueryStatement(sb.toString()); + + final Thread insertThread = + new Thread( + () -> { + for (int i = 0; i < 100; ++i) { + final List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add( + new MeasurementSchema(String.format("m%s", 100 + i), TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + + final Tablet tablet = + new Tablet( + "table8", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (int row = 0; row < 15; row++) { + tablet.addTimestamp(row, timestamp); + tablet.addValue("tag1", row, "tag:" + timestamp); + tablet.addValue("attr1", row, "attr:" + timestamp); + tablet.addValue(String.format("m%s", 100 + i), row, timestamp * 1.0); + timestamp++; + } + + try { + session.insert(tablet); + } catch (final StatementExecutionException | IoTDBConnectionException e) { + throw new RuntimeException(e); + } + tablet.reset(); + } + }); + + final Thread deletionThread = + new Thread( + () -> { + for (int i = 0; i < 100; ++i) { + try { + adminStmt.execute(String.format("alter table db1.table8 drop column m%s", i)); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + }); + + insertThread.start(); + deletionThread.start(); + + insertThread.join(); + deletionThread.join(); + } + } + + @Test + public void testTreeViewTable() throws Exception { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database root.a.b"); + statement.execute("create timeSeries root.a.b.c.S1 int32"); + statement.execute("create timeSeries root.a.b.c.s2 string"); + statement.execute("create timeSeries root.a.b.S1 int32"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database tree_view_db"); + statement.execute("use tree_view_db"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.a.**"); + statement.execute("drop view tree_table"); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create timeSeries root.a.b.d.s1 int32"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use tree_view_db"); + + try { + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.a.**"); + fail(); + } catch (final SQLException e) { + final Set result = + new HashSet<>( + Arrays.asList( + "617: The measurements s1 and S1 share the same lower case when auto detecting type, please check", + "617: The measurements S1 and s1 share the same lower case when auto detecting type, please check")); + assertTrue(result.contains(e.getMessage())); + } + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("drop timeSeries root.a.b.d.s1"); + statement.execute("create device template t1 (S1 boolean, s9 int32)"); + statement.execute("set schema template t1 to root.a.b.d"); + statement.execute("create timeSeries root.a.b.c.f.g.h.S1 int32"); + + // Put schema cache + statement.execute("select S1, s2 from root.a.b.c"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use tree_view_db"); + + try { + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals( + "614: Multiple types encountered when auto detecting type of measurement 'S1', please check", + e.getMessage()); + } + + try { + statement.execute("create view tree_table (tag1 tag, tag2 tag, S1 field) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals( + "614: Multiple types encountered when auto detecting type of measurement 'S1', please check", + e.getMessage()); + } + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create timeSeries root.a.b.e.s1 int32"); + } catch (SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use tree_view_db"); + + // Temporary + try { + statement.execute( + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 boolean from S1) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: The duplicated source measurement S1 is unsupported yet.", e.getMessage()); + } + + try { + statement.execute( + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2, s8 field) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals("528: Measurements not found for s8, cannot auto detect", e.getMessage()); + } + + statement.execute( + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2) as root.a.**"); + statement.execute("alter view tree_table rename to view_table"); + statement.execute("alter view view_table rename column s1 to s11"); + statement.execute("alter view view_table set properties ttl=100"); + statement.execute("comment on view view_table is 'comment'"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show tables details"), + "TableName,TTL(ms),Status,Comment,TableType,", + Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,")); + + TestUtils.assertResultSetEqual( + statement.executeQuery("desc view_table"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "time,TIMESTAMP,TIME,", + "tag1,STRING,TAG,", + "tag2,STRING,TAG,", + "s11,INT32,FIELD,", + "s3,STRING,FIELD,"))); + // Currently we show the device even if all of its measurements does not match, + // the handling logic at query because validate it at fetching will potentially cause a + // lot of time + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from view_table where tag1 = 'b'"), + "tag1,tag2,", + new HashSet<>(Arrays.asList("b,c,", "b,null,", "b,e,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from view_table where tag1 = 'b' and tag2 is null"), + "tag1,tag2,", + Collections.singleton("b,null,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from view_table"), + "count(devices),", + Collections.singleton("3,")); + } + + // Test tree session + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + // Test create & replace + restrict + statement.execute( + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertTrue( + e.getMessage().contains("The 'CreateTableView' is unsupported in tree sql-dialect.")); + } + + // Test permission + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + // Test create & replace + restrict + statement.execute("create user testUser 'testUser'"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection("testUser", "testUser", BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege CREATE ON tree_view_db.view_table", + e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("grant create on tree_view_db.view_table to user testUser"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection("testUser", "testUser", BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege READ_SCHEMA", + e.getMessage()); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("grant read_schema on root.a.** to user testUser"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection("testUser", "testUser", BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.a.**"); + fail(); + } catch (final SQLException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege READ_DATA", + e.getMessage()); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("grant read_data on root.a.** to user testUser"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection("testUser", "testUser", BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.a.**"); + } catch (final SQLException e) { + fail(); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use tree_view_db"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from view_table where tag1 = 'b' and tag2 is null"), + "tag1,tag2,", + Collections.emptySet()); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create view view_table"), + "View,Create View,", + Collections.singleton( + "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.a.**,")); + + // Can also use "show create table" + TestUtils.assertResultSetEqual( + statement.executeQuery("show create table view_table"), + "View,Create View,", + Collections.singleton( + "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.a.**,")); + + statement.execute("create table a ()"); + try { + statement.execute("show create view a"); + fail(); + } catch (final SQLException e) { + assertEquals( + "701: The table a is a base table, does not support show create view.", e.getMessage()); + } + try { + statement.execute("show create view information_schema.tables"); + fail(); + } catch (final SQLException e) { + assertEquals("701: The system view does not support show create.", e.getMessage()); + } + try { + statement.execute("create or replace view a () as root.b.**"); + fail(); + } catch (final SQLException e) { + assertEquals("551: Table 'tree_view_db.a' already exists.", e.getMessage()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java new file mode 100644 index 0000000000000..88b92dbdf0457 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -0,0 +1,1878 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALReader; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSessionRelationalIT { + + @BeforeClass + public static void classSetUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Before + public void setUp() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db2"); + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db3"); + } + } + + @After + public void tearDown() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1"); + session.executeNonQueryStatement("DROP DATABASE IF EXISTS db2"); + session.executeNonQueryStatement("DROP DATABASE IF EXISTS db3"); + } + } + + @AfterClass + public static void classTearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // for manual debugging + public static void main(String[] args) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE \"db1\""); + session.executeNonQueryStatement("CREATE DATABASE \"db2\""); + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table10 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + } + // insert without db + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + long timestamp; + + // no db in session and sql + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table10 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + 0, "tag:" + 0, "attr:" + 0, 0 * 1.0))); + + // specify db in sql + for (long row = 0; row < 15; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db1.table10 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db1.table10 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.table10 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + private static void insertRelationalTabletPerformanceTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table1 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table1", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + } + } + } + + @Test + public void insertAllNullSqlTest() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "create table all_null(color string tag, device_id string tag,city string attribute)"); + try { + session.executeNonQueryStatement("insert into all_null values(null,null,null,null)"); + fail("No exception thrown"); + } catch (StatementExecutionException e) { + assertEquals("701: Timestamp cannot be null", e.getMessage()); + } + session.executeNonQueryStatement("drop table all_null"); + } + } + + @Test + public void insertWrongTimeSqlTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "create table wrong_time(color string tag, device_id string tag,city string attribute)"); + try { + session.executeNonQueryStatement("insert into wrong_time values('aa','bb','cc','dd')"); + fail("No exception thrown"); + } catch (StatementExecutionException e) { + assertEquals( + "701: Input time format aa error. Input like yyyy-MM-dd HH:mm:ss, yyyy-MM-ddTHH:mm:ss or refer to user document for more info.", + e.getMessage()); + } + try { + session.executeNonQueryStatement("insert into wrong_time values(1+1,'bb','cc','dd')"); + fail("No exception thrown"); + } catch (StatementExecutionException e) { + assertEquals("701: Unsupported expression: (1 + 1)", e.getMessage()); + } + try { + session.executeNonQueryStatement("insert into wrong_time values(1.0,'bb','cc','dd')"); + fail("No exception thrown"); + } catch (StatementExecutionException e) { + assertEquals("701: Unsupported expression: 1E0", e.getMessage()); + } + try { + session.executeNonQueryStatement("insert into wrong_time values(true,'bb','cc','dd')"); + fail("No exception thrown"); + } catch (StatementExecutionException e) { + assertEquals("701: Unsupported expression: true", e.getMessage()); + } + session.executeNonQueryStatement("drop table wrong_time"); + } + } + + @Test + public void insertRelationalSqlTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table1 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + long timestamp; + + for (long row = 0; row < 15; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + // without specifying column name + for (long row = 30; row < 40; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + // auto data type conversion + for (long row = 40; row < 50; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES (%d, '%s', '%s', %d)", + row, "tag:" + row, "attr:" + row, row)); + } + + SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(50, cnt); + + // sql cannot create column + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 (tag1, tag2, attr1, m1) VALUES ('%s', '%s', '%s', %f)", + "tag:" + 100, "tag:" + 100, "attr:" + 100, 100 * 1.0))); + + // fewer columns than defined + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES ( '%s', %f)", "attr:" + 100, 100 * 1.0))); + + // more columns than defined + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES ('%s', '%s', '%s', '%s', %f)", + "tag:" + 100, "tag:" + 100, "tag:" + 100, "attr:" + 100, 100 * 1.0))); + + // Invalid conversion - tag column + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES ('%d', '%s', '%s', %f)", + 100, 100, "attr:" + 100, 100 * 1.0))); + + // Invalid conversion - attr column + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES ('%d', '%s', '%s', %f)", + 100, "tag:" + 100, 100, 100 * 1.0))); + + // Invalid conversion - field column + assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + String.format( + "INSERT INTO table1 VALUES ('%d', '%s', '%s', %s)", + 100, "tag:" + 100, "attr:" + 100, "field" + (100 * 1.0)))); + } + } + + @Test + public void partialInsertSQLTest() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + // disable auto-creation only for this test + session.executeNonQueryStatement("SET CONFIGURATION \"enable_auto_create_schema\"=\"false\""); + } + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // the table is missing column "m2" + session.executeNonQueryStatement( + "CREATE TABLE table2_2 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + try { + session.executeNonQueryStatement( + "INSERT INTO table2_2 (time, tag1, attr1, m1, m2) values (1, '1', '1', 1.0, 2.0)"); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "616: Unknown column category for m2. Cannot auto create column.", e.getMessage()); + } + + session.executeNonQueryStatement("CREATE TABLE partial_insert (s1 boolean)"); + try { + session.executeNonQueryStatement( + "insert into partial_insert(time, s1) values (10000,true),(20000,false),(35000,-1.5),(30000,-1),(40000,0),(50000,1),(60000,1.5),(70000,'string'),(80000,'1989-06-15'),(90000,638323200000)"); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "507: Fail to insert measurements [s1] caused by [The BOOLEAN should be true/TRUE, false/FALSE or 0/1]", + e.getMessage()); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from partial_insert order by time"); + long[] timestamps = new long[] {10000, 20000, 40000, 50000}; + Boolean[] values = new Boolean[] {true, false, false, true}; + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rec = dataSet.next(); + assertEquals(timestamps[cnt], rec.getFields().get(0).getLongV()); + assertEquals(values[cnt], rec.getFields().get(1).getBoolV()); + cnt++; + } + assertEquals(4, cnt); + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "SET CONFIGURATION \"enable_auto_create_schema\"=\"true\""); + } + } + } + + @Test + public void partialInsertRelationalTabletTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + // disable auto-creation only for this test + session.executeNonQueryStatement("SET CONFIGURATION \"enable_auto_create_schema\"=\"false\""); + } + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // the table is missing column "m2" + session.executeNonQueryStatement( + "CREATE TABLE table4 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + // the insertion contains "m2" + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("m2", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table4", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + tablet.addValue("m2", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insert(tablet); + } catch (StatementExecutionException e) { + // a partial insertion should be reported + assertEquals( + "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]", + e.getMessage()); + } + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + try { + session.insert(tablet); + } catch (StatementExecutionException e) { + if (!e.getMessage() + .equals( + "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) { + throw e; + } + } + tablet.reset(); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + tablet.addValue("m2", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insert(tablet); + } catch (StatementExecutionException e) { + if (!e.getMessage() + .equals( + "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) { + throw e; + } + } + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + try { + session.insert(tablet); + } catch (StatementExecutionException e) { + if (!e.getMessage() + .equals( + "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) { + throw e; + } + } + tablet.reset(); + } + + SessionDataSet dataSet = session.executeQueryStatement("select * from table4 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + // "m2" should not be present + assertEquals(4, rowRecord.getFields().size()); + cnt++; + } + assertEquals(30, cnt); + + // partial insert is disabled + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='false'"); + int rowIndex = 0; + tablet.addTimestamp(rowIndex, timestamp + rowIndex); + tablet.addValue("tag1", rowIndex, "tag:" + rowIndex); + tablet.addValue("attr1", rowIndex, "attr:" + rowIndex); + tablet.addValue("m1", rowIndex, rowIndex * 1.0); + tablet.addValue("m2", rowIndex, rowIndex * 1.0); + try { + session.insert(tablet); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("616: Missing columns [m2].", e.getMessage()); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\""); + session.executeNonQueryStatement( + "SET CONFIGURATION \"enable_auto_create_schema\"=\"true\""); + } + } + } + + @Test + public void insertRelationalTabletTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table5 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table5", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = session.executeQueryStatement("select * from table5 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void insertNoFieldTest() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS no_field (tag1 string tag)"); + + List schemaList = + Collections.singletonList(new MeasurementSchema("tag1", TSDataType.STRING)); + final List columnTypes = Collections.singletonList(ColumnCategory.TAG); + + Tablet tablet = + new Tablet( + "no_field", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes); + + long timestamp = 0; + for (int row = 0; row < 10; row++) { + tablet.addTimestamp(row, timestamp++); + tablet.addValue("tag1", row, "tag:" + row); + } + try { + session.insert(tablet); + fail("Insert should fail"); + } catch (StatementExecutionException e) { + assertEquals("507: No Field column present, please check the request", e.getMessage()); + } + tablet.reset(); + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from no_field order by time"); + while (dataSet.hasNext()) { + dataSet.next(); + cnt++; + } + assertEquals(0, cnt); + } + } + + @Test + public void insertAllFieldDataTypeMismatchTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS field_wrong_type (tag1 string tag, f1 int32 field, f2 int32 field)"); + + List schemaList = + Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("f1", TSDataType.DOUBLE), + new MeasurementSchema("f2", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD); + + Tablet tablet = + new Tablet( + "field_wrong_type", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes); + + long timestamp = 0; + for (int row = 0; row < 10; row++) { + tablet.addTimestamp(row, timestamp++); + tablet.addValue("tag1", row, "tag:" + row); + tablet.addValue("f1", row, (double) row); + tablet.addValue("f2", row, (double) row); + } + try { + session.insert(tablet); + fail("Insert should fail"); + } catch (StatementExecutionException e) { + assertEquals( + "507: Fail to insert measurements [f1, f2] " + + "caused by [Incompatible data type of column f1: DOUBLE/INT32, " + + "Incompatible data type of column f2: DOUBLE/INT32]", + e.getMessage()); + } + tablet.reset(); + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from field_wrong_type order by time"); + while (dataSet.hasNext()) { + dataSet.next(); + cnt++; + } + assertEquals(0, cnt); + } + } + + @Test + public void insertRelationalTabletWithCacheLeaderTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table5 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table5", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + // cache leader should work for devices that have inserted before + tablet.addValue("tag1", rowIndex, "tag:" + (row - 15)); + tablet.addValue("attr1", rowIndex, "attr:" + (row - 15)); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = session.executeQueryStatement("select * from table5 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals( + "tag:" + (timestamp < 15 ? timestamp : timestamp - 15), + rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals( + "attr:" + (timestamp < 15 ? timestamp : timestamp - 15), + rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void autoCreateNontagColumnTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // only one column in this table, and others should be auto-created + session.executeNonQueryStatement("CREATE TABLE table7 (tag1 string tag)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table7", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + if (tablet.getRowSize() != 0) { + session.insert(tablet); + tablet.reset(); + } + + SessionDataSet dataSet = session.executeQueryStatement("select * from table7 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void autoCreateTableTest() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // no table created here + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table6", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (int row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + } + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLush"); + + for (int row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + } + session.insert(tablet); + tablet.reset(); + + int cnt = 0; + SessionDataSet dataSet = session.executeQueryStatement("select * from table6 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void autoCreateTagColumnTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // only one column in this table, and others should be auto-created + session.executeNonQueryStatement("CREATE TABLE table8 (tag1 string tag)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table8", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (int row = 0; row < 15; row++) { + tablet.addTimestamp(row, timestamp); + tablet.addValue("tag2", row, "tag:" + timestamp); + tablet.addValue("attr1", row, "attr:" + timestamp); + tablet.addValue("m1", row, timestamp * 1.0); + timestamp++; + } + + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = session.executeQueryStatement("select * from table8 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long t = rowRecord.getFields().get(0).getLongV(); + // tag 1 should be null + assertNull(rowRecord.getFields().get(1).getDataType()); + assertEquals("tag:" + t, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals("attr:" + t, rowRecord.getFields().get(3).getBinaryV().toString()); + assertEquals(t * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + + session.executeNonQueryStatement("FLush"); + + for (int row = 0; row < 15; row++) { + tablet.addTimestamp(row, timestamp); + tablet.addValue("tag2", row, "tag:" + timestamp); + tablet.addValue("attr1", row, "attr:" + timestamp); + tablet.addValue("m1", row, timestamp * 1.0); + timestamp++; + } + + session.insert(tablet); + tablet.reset(); + + dataSet = session.executeQueryStatement("select * from table8 order by time"); + cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long t = rowRecord.getFields().get(0).getLongV(); + // tag 1 should be null + assertNull(rowRecord.getFields().get(1).getDataType()); + assertEquals("tag:" + t, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals("attr:" + t, rowRecord.getFields().get(3).getBinaryV().toString()); + assertEquals(t * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void autoAdjustTagTest() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // the tag order in the table is (tag1, tag2) + session.executeNonQueryStatement( + "CREATE TABLE table9 (tag1 string tag, tag2 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + // the tag order in the row is (tag2, tag1) + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD); + List fieldtags = IMeasurementSchema.getMeasurementNameList(schemaList); + List dataTypes = IMeasurementSchema.getDataTypeList(schemaList); + + long timestamp = 0; + Tablet tablet = new Tablet("table9", fieldtags, dataTypes, columnTypes, 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag2", rowIndex, "tag2:" + row); + tablet.addValue("tag1", rowIndex, "tag1:" + row); + tablet.addValue("attr1", rowIndex, "attr1:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + } + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag2", rowIndex, "tag2:" + row); + tablet.addValue("tag1", rowIndex, "tag1:" + row); + tablet.addValue("attr1", rowIndex, "attr1:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + } + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = session.executeQueryStatement("select * from table9 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag1:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("tag2:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals("attr1:" + timestamp, rowRecord.getFields().get(3).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void insertRelationalSqlWithoutDBTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table10 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + } + // insert without db + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + long timestamp; + + // no db in session and sql + try { + session.executeNonQueryStatement( + String.format( + "INSERT INTO table10 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("701: database is not specified", e.getMessage()); + } + + // specify db in sql + for (long row = 0; row < 15; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db1.table10 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db1.table10 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.table10 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void insertRelationalSqlAnotherDBTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table11 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + } + // use db2 but insert db1 + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + long timestamp; + session.executeNonQueryStatement("USE \"db2\""); + + // specify db in sql + for (long row = 0; row < 15; row++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db1.table11 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + // check case sensitivity + session.executeNonQueryStatement( + String.format( + "INSERT INTO DB1.TaBle11 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + row, "tag:" + row, "attr:" + row, row * 1.0)); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.table11 order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("tag:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void insertNonExistTableTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + + try { + session.executeNonQueryStatement( + String.format( + "INSERT INTO table13 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("550: Table 'db1.table13' does not exist.", e.getMessage()); + } + + try { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db2.table13 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("550: Table 'db2.table13' does not exist.", e.getMessage()); + } + } + } + + @Test + public void insertNonExistDBTest() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + + try { + session.executeNonQueryStatement( + String.format( + "INSERT INTO db3.table13 (time, tag1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("550: Table 'db3.table13' does not exist.", e.getMessage()); + } + } + } + + @Test + public void insertWithoutFieldTest() throws IoTDBConnectionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement("create table tb (a string tag, b string field)"); + session.executeNonQueryStatement("insert into tb(a) values ('w')"); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("507: No Field column present, please check the request", e.getMessage()); + } + } + + private void testOneCastWithTablet( + TSDataType from, TSDataType to, int testNum, boolean partialInsert) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // create a column with type of "to" + session.executeNonQueryStatement( + "CREATE TABLE table" + + testNum + + " (tag1 string tag," + + "m1 " + + to.toString() + + " field)"); + if (partialInsert) { + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='true'"); + } else { + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='false'"); + } + + // insert a tablet with type of "from" + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", from)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + "table" + testNum, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + tablet.addTimestamp(0, 0); + tablet.addTimestamp(1, 1); + tablet.addValue(0, 0, "d1"); + tablet.addValue(1, 0, "d1"); + // the field in the first row is null + tablet.addValue("m1", 1, genValue(from, 1)); + if (to.isCompatible(from)) { + // can cast, insert and check the result + session.insert(tablet); + // time, tag1, m1 + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + if (to == TSDataType.BLOB) { + assertEquals(genValue(to, 1), rec.getFields().get(2).getBinaryV()); + } else if (to == TSDataType.DATE) { + assertEquals(genValue(to, 1), rec.getFields().get(2).getDateV()); + } else { + assertEquals(genValue(to, 1).toString(), rec.getFields().get(2).toString()); + } + assertFalse(dataSet.hasNext()); + } else { + if (partialInsert) { + // cannot cast, but partial insert + try { + session.insert(tablet); + fail("Exception expected: from=" + from + ", to=" + to); + } catch (StatementExecutionException e) { + assertEquals( + "507: Fail to insert measurements [m1] caused by [Incompatible data type of column m1: " + + from + + "/" + + to + + "]", + e.getMessage()); + } + // time, tag1, m1 + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + assertFalse(dataSet.hasNext()); + } else { + // cannot cast, expect an exception + try { + session.insert(tablet); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "614: Incompatible data type of column m1: " + from + "/" + to, e.getMessage()); + } + } + } + + session.executeNonQueryStatement("DROP TABLE table" + testNum); + } + } + + private void testOneCastWithRow( + TSDataType from, TSDataType to, int testNum, boolean partialInsert) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // create a column with type of "to" + session.executeNonQueryStatement( + "CREATE TABLE table" + + testNum + + " (tag1 string tag," + + "m1 " + + to.toString() + + " field)"); + if (partialInsert) { + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='true'"); + } else { + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='false'"); + } + + // insert a tablet with type of "from" + String sql = + String.format( + "INSERT INTO table" + + testNum + + " (time, tag1, m1) VALUES (0, 'd1', null),(1,'d1', %s)", + genValue(from, 1)); + if (to.isCompatible(from)) { + // can cast, insert and check the result + session.executeNonQueryStatement(sql); + // time, tag1, m1 + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + if (to == TSDataType.BLOB) { + assertEquals(genValue(to, 1), rec.getFields().get(2).getBinaryV()); + } else if (to == TSDataType.DATE) { + assertEquals(genValue(to, 1), rec.getFields().get(2).getDateV()); + } else { + assertEquals(genValue(to, 1).toString(), rec.getFields().get(2).toString()); + } + assertFalse(dataSet.hasNext()); + } else { + if (partialInsert) { + // cannot cast, but partial insert + try { + session.executeNonQueryStatement(sql); + fail("Exception expected: from=" + from + ", to=" + to); + } catch (StatementExecutionException e) { + assertEquals( + "507: Fail to insert measurements [m1] caused by [Incompatible data type of column m1: " + + from + + "/" + + to + + "]", + e.getMessage()); + } + // time, tag1, m1 + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + assertFalse(dataSet.hasNext()); + } else { + // cannot cast, expect an exception + try { + session.executeNonQueryStatement(sql); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "614: Incompatible data type of column m1: " + from + "/" + to, e.getMessage()); + } + } + } + + session.executeNonQueryStatement("DROP TABLE table" + testNum); + } + } + + @SuppressWarnings("SameParameterValue") + private Object genValue(TSDataType dataType, int i) { + switch (dataType) { + case INT32: + return i; + case DATE: + return LocalDate.ofEpochDay(i); + case TIMESTAMP: + case INT64: + return (long) i; + case BOOLEAN: + return i % 2 == 0; + case FLOAT: + return i * 1.0f; + case DOUBLE: + return i * 1.0; + case STRING: + case TEXT: + case BLOB: + return new Binary(Integer.toString(i), StandardCharsets.UTF_8); + case UNKNOWN: + case VECTOR: + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } + + @Test + public void insertRelationalTabletWithAutoCastTest() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 14; + Set dataTypes = new HashSet<>(); + Collections.addAll(dataTypes, TSDataType.values()); + dataTypes.remove(TSDataType.VECTOR); + dataTypes.remove(TSDataType.UNKNOWN); + + for (TSDataType from : dataTypes) { + for (TSDataType to : dataTypes) { + System.out.println("from: " + from + ", to: " + to); + testOneCastWithTablet(from, to, testNum, false); + System.out.println("partial insert"); + testOneCastWithTablet(from, to, testNum, true); + } + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\""); + } + } + + @Test + public void deleteTableAndWriteDifferentTypeTest() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 15; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE db1"); + + session.executeNonQueryStatement( + "CREATE TABLE table" + testNum + " (tag1 string tag, m1 int32 field)"); + session.executeNonQueryStatement( + "INSERT INTO table" + testNum + " (time, tag1, m1) VALUES (1, 'd1', 1)"); + + session.executeNonQueryStatement("DROP TABLE table" + testNum); + + session.executeNonQueryStatement( + "CREATE TABLE table" + testNum + " (tag1 string tag, m1 double field)"); + session.executeNonQueryStatement( + "INSERT INTO table" + testNum + " (time, tag1, m1) VALUES (2, 'd2', 2)"); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + assertEquals(2, rec.getFields().get(0).getLongV()); + assertEquals("d2", rec.getFields().get(1).toString()); + assertEquals(2.0, rec.getFields().get(2).getDoubleV(), 0.1); + assertFalse(dataSet.hasNext()); + } + } + + @Test + public void dropTableOfTheSameNameTest() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 16; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE db1"); + + session.executeNonQueryStatement( + "CREATE TABLE db1.table" + testNum + " (tag1 string tag, m1 int32 field)"); + session.executeNonQueryStatement( + "INSERT INTO db1.table" + testNum + " (time, tag1, m1) VALUES (1, 'd1', 1)"); + + session.executeNonQueryStatement( + "CREATE TABLE db2.table" + testNum + " (tag1 string tag, m1 double field)"); + session.executeNonQueryStatement( + "INSERT INTO db2.table" + testNum + " (time, tag1, m1) VALUES (2, 'd2', 2)"); + + session.executeNonQueryStatement("DROP TABLE db2.table" + testNum); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertEquals(1, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + try { + session.executeQueryStatement("select * from db2.table" + testNum + " order by time"); + fail("expected exception"); + } catch (StatementExecutionException e) { + assertEquals("550: Table 'db2.table16' does not exist.", e.getMessage()); + } + } + } + + @Test + public void insertRelationalRowWithAutoCastTest() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 17; + Set dataTypes = new HashSet<>(); + Collections.addAll(dataTypes, TSDataType.values()); + dataTypes.remove(TSDataType.VECTOR); + dataTypes.remove(TSDataType.UNKNOWN); + + for (TSDataType from : dataTypes) { + for (TSDataType to : dataTypes) { + System.out.println("from: " + from + ", to: " + to); + testOneCastWithTablet(from, to, testNum, false); + System.out.println("partial insert"); + testOneCastWithTablet(from, to, testNum, true); + } + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\""); + } + } + + @Test + public void insertMinMaxTimeTest() throws IoTDBConnectionException, StatementExecutionException { + try { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='false'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("USE db1"); + session.executeNonQueryStatement("CREATE TABLE test_insert_min_max (id1 TAG, s1 INT32)"); + + session.executeNonQueryStatement( + String.format( + "INSERT INTO test_insert_min_max(time, id1, s1) VALUES (%d, 'd1', 1)", + Long.MIN_VALUE)); + session.executeNonQueryStatement( + String.format( + "INSERT INTO test_insert_min_max(time, id1, s1) VALUES (%d, 'd1', 1)", + Long.MAX_VALUE)); + + SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM test_insert_min_max"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getFields().get(0).getLongV()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("FLUSH"); + dataSet = session.executeQueryStatement("SELECT * FROM test_insert_min_max"); + record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getFields().get(0).getLongV()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='true'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + } + } + + @Test + public void loadMinMaxTimeAlignedTest() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + File file = new File("target", "test.tsfile"); + TableSchema tableSchema = + new TableSchema( + "load_min_max", + Arrays.asList("id1", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { + Tablet tablet = + new Tablet( + Arrays.asList("id1", "s1"), Arrays.asList(TSDataType.STRING, TSDataType.INT32)); + tablet.addTimestamp(0, Long.MIN_VALUE); + tablet.addTimestamp(1, Long.MAX_VALUE); + tablet.addValue(0, 0, "d1"); + tablet.addValue(1, 0, "d1"); + tablet.addValue(0, 1, 1); + tablet.addValue(1, 1, 1); + writer.write(tablet); + } + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE db1"); + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='false'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("LOAD \'" + file.getAbsolutePath() + "\'"); + + SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM load_min_max"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getFields().get(0).getLongV()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='false'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + file.delete(); + } + } + + @Test + public void autoCreateTagColumnTest2() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 18; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // only one column in this table, and others should be auto-created + session.executeNonQueryStatement( + "CREATE TABLE table" + testNum + " (tag1 string tag, s1 text field)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table" + testNum, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (int row = 0; row < 15; row++) { + tablet.addTimestamp(row, timestamp); + tablet.addValue("tag2", row, "string"); + tablet.addValue("s2", row, timestamp); + timestamp++; + } + + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long t = rowRecord.getFields().get(0).getLongV(); + // tag 1 should be null + assertNull(rowRecord.getFields().get(1).getDataType()); + // s1 should be null + assertNull(rowRecord.getFields().get(2).getDataType()); + assertEquals("string", rowRecord.getFields().get(3).getBinaryV().toString()); + assertEquals(t, rowRecord.getFields().get(4).getLongV()); + cnt++; + } + assertEquals(15, cnt); + + session.executeNonQueryStatement("FLush"); + + for (int row = 0; row < 15; row++) { + tablet.addTimestamp(row, timestamp); + tablet.addValue("tag2", row, "string"); + tablet.addValue("s2", row, timestamp); + timestamp++; + } + + session.insert(tablet); + tablet.reset(); + + dataSet = session.executeQueryStatement("select * from table" + testNum + " order by time"); + cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long t = rowRecord.getFields().get(0).getLongV(); + // tag 1 should be null + assertNull(rowRecord.getFields().get(1).getDataType()); + // s1 should be null + assertNull(rowRecord.getFields().get(2).getDataType()); + assertEquals("string", rowRecord.getFields().get(3).getBinaryV().toString()); + assertEquals(t, rowRecord.getFields().get(4).getLongV()); + cnt++; + } + assertEquals(30, cnt); + } + } + + @Test + public void testAttrColumnRemoved() + throws IoTDBConnectionException, StatementExecutionException, IOException { + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); + EnvFactory.getEnv().initClusterEnvironment(); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("create database if not exists db1"); + session.executeNonQueryStatement("use db1"); + session.executeNonQueryStatement( + "CREATE TABLE remove_attr_col (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)"); + + // insert tablet to WAL + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "remove_attr_col", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes); + + for (int rowIndex = 0; rowIndex < 10; rowIndex++) { + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("tag1", rowIndex, "tag:1"); + tablet.addValue("attr1", rowIndex, "attr:" + timestamp); + tablet.addValue("m1", rowIndex, timestamp * 1.0); + timestamp++; + } + session.insert(tablet); + tablet.reset(); + + // insert records to WAL + session.executeNonQueryStatement( + "INSERT INTO remove_attr_col (time, tag1, attr1, m1) VALUES (10, 'tag:1', 'attr:10', 10.0)"); + + // check WAL + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String walNodeDir = dataNodeWrapper.getWalDir() + File.separator + "0"; + File[] walFiles = new File(walNodeDir).listFiles(f -> f.getName().endsWith(".wal")); + if (walFiles != null && walFiles.length > 0) { + File walFile = walFiles[0]; + WALEntry entry; + try (WALReader walReader = new WALReader(walFile)) { + entry = walReader.next(); + RelationalInsertTabletNode tabletNode = (RelationalInsertTabletNode) entry.getValue(); + assertTrue( + Arrays.stream(tabletNode.getColumnCategories()) + .noneMatch(c -> c == TsTableColumnCategory.ATTRIBUTE)); + + entry = walReader.next(); + RelationalInsertRowsNode rowsNode = (RelationalInsertRowsNode) entry.getValue(); + assertTrue( + Arrays.stream(rowsNode.getInsertRowNodeList().get(0).getColumnCategories()) + .noneMatch(c -> c == TsTableColumnCategory.ATTRIBUTE)); + return; + } + } + } + } finally { + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv().initClusterEnvironment(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java new file mode 100644 index 0000000000000..b3014ff56ee62 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.read.common.RowRecord; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableModelSessionIT { + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testUseDatabase() { + final String[] table1Names = new String[] {"table1"}; + final String[] table1ttls = new String[] {"3600000"}; + + final String[] table2Names = new String[] {"table2"}; + final String[] table2ttls = new String[] {"6600000"}; + + try (final ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB("test2")) { + + session.executeNonQueryStatement("CREATE DATABASE test1"); + session.executeNonQueryStatement("CREATE DATABASE test2"); + + // or use full qualified table name + session.executeNonQueryStatement( + "create table test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + + session.executeNonQueryStatement( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD) with (TTL=6600000)"); + + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + final RowRecord rowRecord = dataSet.next(); + assertEquals(table2Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table2ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table2Names.length, cnt); + } + + session.executeNonQueryStatement("use test1"); + + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + final RowRecord rowRecord = dataSet.next(); + assertEquals(table1Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table1ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table1Names.length, cnt); + } + + } catch (final IoTDBConnectionException | StatementExecutionException e) { + fail(e.getMessage()); + } + } + + @Test + public void testCreateSessionWithCapitalDB() { + final String[] table2Names = new String[] {"table2"}; + final String[] table2ttls = new String[] {"6600000"}; + + try (final ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB("TEST2")) { + + session.executeNonQueryStatement("CREATE DATABASE test1"); + session.executeNonQueryStatement("CREATE DATABASE test2"); + + // or use full qualified table name + session.executeNonQueryStatement( + "create table test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + + session.executeNonQueryStatement( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD) with (TTL=6600000)"); + + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + final RowRecord rowRecord = dataSet.next(); + assertEquals(table2Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table2ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table2Names.length, cnt); + } + + } catch (final IoTDBConnectionException | StatementExecutionException e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBInsertTableSessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBInsertTableSessionPoolIT.java new file mode 100644 index 0000000000000..c92d09b4dbba6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBInsertTableSessionPoolIT.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session.pool; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.nio.charset.Charset; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBInsertTableSessionPoolIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + ITableSessionPool sessionPool = EnvFactory.getEnv().getTableSessionPool(1); + try (final ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement("create database if not exists test"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testPartialInsertTablet() { + final ITableSessionPool sessionPool = EnvFactory.getEnv().getTableSessionPool(1); + try (final ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement("use \"test\""); + session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + session.executeNonQueryStatement( + "create table sg6 (tag1 string tag, s1 int64 field, s2 int64 field)"); + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + "sg6", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 300); + long timestamp = 0; + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 4; s++) { + long value = timestamp; + if (s == 0) { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, "d1"); + } else { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + } + timestamp++; + } + timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 4; s++) { + long value = timestamp; + if (s == 0) { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, "d1"); + } else { + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + } + timestamp++; + } + try { + session.insert(tablet); + } catch (Exception e) { + if (!e.getMessage().contains("507")) { + fail(e.getMessage()); + } + } finally { + session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + } + try (SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM sg6")) { + assertEquals(4, dataSet.getColumnNames().size()); + assertEquals("time", dataSet.getColumnNames().get(0)); + assertEquals("tag1", dataSet.getColumnNames().get(1)); + assertEquals("s1", dataSet.getColumnNames().get(2)); + assertEquals("s2", dataSet.getColumnNames().get(3)); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long time = rowRecord.getFields().get(0).getLongV(); + assertEquals(time, rowRecord.getFields().get(2).getLongV()); + assertEquals(time, rowRecord.getFields().get(3).getLongV()); + cnt++; + } + Assert.assertEquals(200, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testInsertKeyword() throws IoTDBConnectionException, StatementExecutionException { + ITableSessionPool sessionPool = EnvFactory.getEnv().getTableSessionPool(1); + try (final ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "create table table20 (" + + "device_id string tag," + + "attribute STRING ATTRIBUTE," + + "boolean boolean FIELD," + + "int32 int32 FIELD," + + "int64 int64 FIELD," + + "float float FIELD," + + "double double FIELD," + + "text text FIELD," + + "string string FIELD," + + "blob blob FIELD," + + "timestamp01 timestamp FIELD," + + "date date FIELD)"); + + List schemas = new ArrayList<>(); + schemas.add(new MeasurementSchema("device_id", TSDataType.STRING)); + schemas.add(new MeasurementSchema("attribute", TSDataType.STRING)); + schemas.add(new MeasurementSchema("boolean", TSDataType.BOOLEAN)); + schemas.add(new MeasurementSchema("int32", TSDataType.INT32)); + schemas.add(new MeasurementSchema("int64", TSDataType.INT64)); + schemas.add(new MeasurementSchema("float", TSDataType.FLOAT)); + schemas.add(new MeasurementSchema("double", TSDataType.DOUBLE)); + schemas.add(new MeasurementSchema("text", TSDataType.TEXT)); + schemas.add(new MeasurementSchema("string", TSDataType.STRING)); + schemas.add(new MeasurementSchema("blob", TSDataType.BLOB)); + schemas.add(new MeasurementSchema("timestamp", TSDataType.TIMESTAMP)); + schemas.add(new MeasurementSchema("date", TSDataType.DATE)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "table20", + IMeasurementSchema.getMeasurementNameList(schemas), + schemas.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()), + columnTypes, + 10); + + for (long row = 0; row < 10; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("device_id", rowIndex, "1"); + tablet.addValue("attribute", rowIndex, "1"); + tablet.addValue("boolean", rowIndex, true); + tablet.addValue("int32", rowIndex, Integer.valueOf("1")); + tablet.addValue("int64", rowIndex, Long.valueOf("1")); + tablet.addValue("float", rowIndex, Float.valueOf("1.0")); + tablet.addValue("double", rowIndex, Double.valueOf("1.0")); + tablet.addValue("text", rowIndex, "true"); + tablet.addValue("string", rowIndex, "true"); + tablet.addValue("blob", rowIndex, new Binary("iotdb", Charset.defaultCharset())); + tablet.addValue("timestamp", rowIndex, 1L); + tablet.addValue("date", rowIndex, LocalDate.parse("2024-08-15")); + } + session.insert(tablet); + + SessionDataSet rs1 = + session.executeQueryStatement( + "select time, device_id, attribute, boolean, int32, int64, float, double, text, string, blob, timestamp, date from table20 order by time"); + for (int i = 0; i < 10; i++) { + RowRecord rec = rs1.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + assertEquals("1", rec.getFields().get(1).getStringValue()); + assertEquals("1", rec.getFields().get(2).getStringValue()); + assertTrue(rec.getFields().get(3).getBoolV()); + assertEquals(1, rec.getFields().get(4).getIntV()); + assertEquals(1, rec.getFields().get(5).getLongV()); + assertEquals(1.0, rec.getFields().get(6).getFloatV(), 0.001); + assertEquals(1.0, rec.getFields().get(7).getDoubleV(), 0.001); + assertEquals("true", rec.getFields().get(8).getStringValue()); + assertEquals("true", rec.getFields().get(9).getStringValue()); + assertEquals("0x696f746462", rec.getFields().get(10).getStringValue()); + assertEquals(1, rec.getFields().get(11).getLongV()); + assertEquals("20240815", rec.getFields().get(12).getStringValue()); + } + assertFalse(rs1.hasNext()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBTableModelSessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBTableModelSessionPoolIT.java new file mode 100644 index 0000000000000..3e2cba94c4c7b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBTableModelSessionPoolIT.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session.pool; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.read.common.RowRecord; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableModelSessionPoolIT { + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testUseDatabase() { + + final String[] table1Names = new String[] {"table1"}; + final String[] table1ttls = new String[] {"3600000"}; + + final String[] table2Names = new String[] {"table2"}; + final String[] table2ttls = new String[] {"6600000"}; + + ITableSessionPool sessionPool = EnvFactory.getEnv().getTableSessionPool(1); + try (final ITableSession session = sessionPool.getSession()) { + + session.executeNonQueryStatement("CREATE DATABASE test1"); + session.executeNonQueryStatement("CREATE DATABASE test2"); + + session.executeNonQueryStatement("use test2"); + + // or use full qualified table name + session.executeNonQueryStatement( + "create table test1.table1(region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)"); + + session.executeNonQueryStatement( + "create table table2(region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD, speed DOUBLE FIELD) with (TTL=6600000)"); + + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + final RowRecord rowRecord = dataSet.next(); + assertEquals(table2Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table2ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table2Names.length, cnt); + } + + } catch (final IoTDBConnectionException | StatementExecutionException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final ITableSession session = sessionPool.getSession()) { + // current session's database is still test2 + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(table2Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table2ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table2Names.length, cnt); + } + + } catch (final IoTDBConnectionException | StatementExecutionException e) { + fail(e.getMessage()); + } finally { + sessionPool.close(); + } + + // specify database in constructor + sessionPool = EnvFactory.getEnv().getTableSessionPool(1, "test1"); + + try (final ITableSession session = sessionPool.getSession()) { + + // current session's database is test1 + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(table1Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table1ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table1Names.length, cnt); + } + + // change database to test2 + session.executeNonQueryStatement("use test2"); + + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(table2Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table2ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table2Names.length, cnt); + } + + } catch (final IoTDBConnectionException | StatementExecutionException e) { + fail(e.getMessage()); + } + + // after putting back, the session's database should be changed back to default test1 + try (final ITableSession session = sessionPool.getSession()) { + + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + int cnt = 0; + assertEquals(showTablesColumnHeaders.size(), dataSet.getColumnNames().size()); + for (int i = 0; i < showTablesColumnHeaders.size(); i++) { + assertEquals( + showTablesColumnHeaders.get(i).getColumnName(), dataSet.getColumnNames().get(i)); + } + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(table1Names[cnt], rowRecord.getFields().get(0).getStringValue()); + assertEquals(table1ttls[cnt], rowRecord.getFields().get(1).getStringValue()); + cnt++; + } + assertEquals(table1Names.length, cnt); + } + + } catch (final IoTDBConnectionException | StatementExecutionException e) { + fail(e.getMessage()); + } finally { + sessionPool.close(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java index 1bf996f6f950f..2340cf3f326b2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java @@ -30,6 +30,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -291,7 +292,7 @@ private void insertTabletWithAlignedTimeseriesMethod(ISession session) throws IoTDBConnectionException, StatementExecutionException { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); @@ -299,19 +300,19 @@ private void insertTabletWithAlignedTimeseriesMethod(ISession session) long timestamp = 0; for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, row * 10 + 1L); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (int) (row * 10 + 2)); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, row * 10 + 1L); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (int) (row * 10 + 2)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertAlignedTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertAlignedTablet(tablet); tablet.reset(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java index 55c93735c5c71..eef6614aa624f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java @@ -35,6 +35,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -274,7 +275,7 @@ public void alignByDeviceTest() { private void insertTablet(ISession session, String deviceId) throws IoTDBConnectionException, StatementExecutionException { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE)); @@ -282,41 +283,37 @@ private void insertTablet(ISession session, String deviceId) Tablet tablet = new Tablet(deviceId, schemaList, 100); for (long time = 0; time < 100; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); long value = 0; tablet.addTimestamp(rowIndex, time); for (int s = 0; s < 3; s++) { - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); value++; } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet); tablet.reset(); } } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = i; + tablet.addValue(row, i, (long) i); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet); tablet.reset(); } } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -374,7 +371,7 @@ private void insertRecords(ISession session, List deviceIdList) private void insertMultiTablets(ISession session, List deviceIdList) throws IoTDBConnectionException, StatementExecutionException { long timePartition = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); @@ -387,12 +384,12 @@ private void insertMultiTablets(ISession session, List deviceIdList) for (long time = 0; time < 10 * timePartition; time += timePartition / 10) { for (Tablet tablet : tabletMap.values()) { long value = 0; - tablet.addTimestamp(tablet.rowSize, time); + int rowSize = tablet.getRowSize(); + tablet.addTimestamp(rowSize, time); for (int s = 0; s < 3; s++) { - tablet.addValue(schemaList.get(s).getMeasurementId(), tablet.rowSize, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowSize, value); value++; } - tablet.rowSize++; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionDisableMemControlIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionDisableMemControlIT.java index 98d9f8106a0e5..a24e2c20c2615 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionDisableMemControlIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionDisableMemControlIT.java @@ -33,6 +33,7 @@ import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -85,7 +86,7 @@ public void insertPartialTabletTest() { session.createTimeseries( "root.sg.d.s3", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); } - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT)); @@ -95,12 +96,12 @@ public void insertPartialTabletTest() { long timestamp = System.currentTimeMillis(); for (long row = 0; row < 15; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue("s1", rowIndex, 1L); tablet.addValue("s2", rowIndex, 1D); tablet.addValue("s3", rowIndex, new Binary("1", TSFileConfig.STRING_CHARSET)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { try { session.insertTablet(tablet, true); } catch (StatementExecutionException e) { @@ -113,7 +114,7 @@ public void insertPartialTabletTest() { timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { try { session.insertTablet(tablet); } catch (StatementExecutionException e) { @@ -164,7 +165,7 @@ public void insertPartialAlignedTabletTest() { null, null, null); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT)); @@ -174,12 +175,12 @@ public void insertPartialAlignedTabletTest() { long timestamp = System.currentTimeMillis(); for (long row = 0; row < 15; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue("s1", rowIndex, 1L); tablet.addValue("s2", rowIndex, 1D); tablet.addValue("s3", rowIndex, new Binary("1", TSFileConfig.STRING_CHARSET)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { try { session.insertAlignedTablet(tablet, true); } catch (StatementExecutionException e) { @@ -192,7 +193,7 @@ public void insertPartialAlignedTabletTest() { timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { try { session.insertAlignedTablet(tablet); } catch (StatementExecutionException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index f101a833fbb3b..5fdf733578f75 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -32,7 +33,10 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -40,11 +44,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -355,4 +361,154 @@ public void insertAlignedRecordsOfOneDeviceNullTest() { fail(e.getMessage()); } } + + @Test + public void insertTabletNullTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + + String deviceId = "root.sg1.clsu.d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.INT32)), + 3); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, null); + tablet.addValue("s2", 0, null); + tablet.addTimestamp(1, 400); + tablet.addValue("s1", 1, null); + tablet.addValue("s2", 1, null); + tablet.addTimestamp(2, 500); + tablet.addValue("s1", 2, null); + tablet.addValue("s2", 2, null); + session.insertTablet(tablet); + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + session.executeNonQueryStatement("flush"); + nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void insertAlignedTabletNullTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.INT32)), + 3); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, null); + tablet.addValue("s2", 0, null); + tablet.addTimestamp(1, 400); + tablet.addValue("s1", 1, null); + tablet.addValue("s2", 1, null); + tablet.addTimestamp(2, 500); + tablet.addValue("s1", 2, null); + tablet.addValue("s2", 2, null); + session.insertAlignedTablet(tablet); + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + session.executeNonQueryStatement("flush"); + nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + for (DataNodeWrapper dn : EnvFactory.getEnv().getDataNodeWrapperList()) { + File dir = + new File( + dn.getDataDir() + + File.separator + + "datanode" + + File.separator + + "data" + + File.separator + + "sequence" + + File.separator + + "root.sg1" + + File.separator + + "1" + + File.separator + + "0"); + if (dir.exists() && dir.isDirectory()) { + File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + assertFalse(file.getName().endsWith("broken")); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void insertTabletNullMeasurementTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema(null, TSDataType.INT32)), + 1); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, true); + tablet.addValue(null, 0, 1); + session.insertAlignedTablet(tablet); + fail(); + } catch (Exception e) { + Assert.assertEquals("measurement should be non null value", e.getMessage()); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema(null, TSDataType.INT32)), + 1); + tablet.addTimestamp(0, 300); + tablet.addValue(0, 0, true); + tablet.addValue(0, 1, 1); + session.insertAlignedTablet(tablet); + fail(); + } catch (Exception e) { + Assert.assertEquals("measurement should be non null value", e.getMessage()); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema(null, TSDataType.INT32)), + 1); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, true); + // doesn't insert 2nd measurement + session.insertAlignedTablet(tablet); + fail(); + } catch (Exception e) { + Assert.assertEquals("measurement should be non null value", e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertWithTriggerExecutionIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertWithTriggerExecutionIT.java index ab482592eac85..ef217d4f57a44 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertWithTriggerExecutionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertWithTriggerExecutionIT.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.AfterClass; import org.junit.Assert; @@ -297,7 +298,7 @@ public void testFireTimesOfStatefulTrigger() { private void insertTablet(ISession session, String device, List measurementList) throws IoTDBConnectionException, StatementExecutionException { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); measurementList.forEach( measurement -> schemaList.add(new MeasurementSchema(measurement, TSDataType.INT32))); @@ -305,17 +306,17 @@ private void insertTablet(ISession session, String device, List measurem long timestamp = 1; for (int i = 0; i < rows; i++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); measurementList.forEach(measurement -> tablet.addValue(measurement, rowIndex, 1)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java index 0a89ee56ded15..543a3707c0270 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java @@ -20,6 +20,7 @@ import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.protocol.thrift.OperationType; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; @@ -36,24 +37,34 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; import java.security.SecureRandom; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -62,6 +73,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -70,18 +82,32 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@SuppressWarnings({"ThrowFromFinallyBlock", "ResultOfMethodCallIgnored"}) @RunWith(IoTDBTestRunner.class) public class IoTDBSessionSimpleIT { private static Logger LOGGER = LoggerFactory.getLogger(IoTDBSessionSimpleIT.class); - @Before - public void setUp() throws Exception { + private static final String[] databasesToClear = new String[] {"root.sg", "root.sg1"}; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } @After - public void tearDown() throws Exception { + public void tearDown() { + for (String database : databasesToClear) { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("DELETE DATABASE " + database); + } catch (Exception ignored) { + + } + } + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } @@ -89,7 +115,7 @@ public void tearDown() throws Exception { @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertPartialTabletTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT)); @@ -99,19 +125,19 @@ public void insertPartialTabletTest() { long timestamp = System.currentTimeMillis(); for (long row = 0; row < 15; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue("s1", rowIndex, 1L); tablet.addValue("s2", rowIndex, 1D); tablet.addValue("s3", rowIndex, new Binary("1", TSFileConfig.STRING_CHARSET)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -135,7 +161,7 @@ public void insertPartialTabletsTest() { session.createTimeseries( "root.sg.d2.s2", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT)); @@ -152,25 +178,25 @@ public void insertPartialTabletsTest() { long timestamp = System.currentTimeMillis(); for (long row = 0; row < 15; row++) { - int rowIndex1 = tablet1.rowSize++; + int rowIndex1 = tablet1.getRowSize(); tablet1.addTimestamp(rowIndex1, timestamp); tablet1.addValue("s1", rowIndex1, 1L); tablet1.addValue("s2", rowIndex1, 1D); tablet1.addValue("s3", rowIndex1, new Binary("1", TSFileConfig.STRING_CHARSET)); - int rowIndex2 = tablet2.rowSize++; + int rowIndex2 = tablet2.getRowSize(); tablet2.addTimestamp(rowIndex2, timestamp); tablet2.addValue("s1", rowIndex2, 1L); tablet2.addValue("s2", rowIndex2, 1D); tablet2.addValue("s3", rowIndex2, new Binary("1", TSFileConfig.STRING_CHARSET)); - int rowIndex3 = tablet3.rowSize++; + int rowIndex3 = tablet3.getRowSize(); tablet3.addTimestamp(rowIndex3, timestamp); tablet3.addValue("s1", rowIndex3, 1L); tablet3.addValue("s2", rowIndex3, 1D); tablet3.addValue("s3", rowIndex3, new Binary("1", TSFileConfig.STRING_CHARSET)); - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { session.insertTablets(tabletMap); tablet1.reset(); tablet2.reset(); @@ -179,7 +205,7 @@ public void insertPartialTabletsTest() { timestamp++; } - if (tablet1.rowSize != 0) { + if (tablet1.getRowSize() != 0) { session.insertTablets(tabletMap); tablet1.reset(); tablet2.reset(); @@ -321,7 +347,7 @@ public void insertByObjAndNotInferTypeTest() { expected.add(TSDataType.TEXT.name()); Set actual = new HashSet<>(); - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root.**"); + SessionDataSet dataSet = session.executeQueryStatement("show timeseries root.sg1.**"); while (dataSet.hasNext()) { actual.add(dataSet.next().getFields().get(3).getStringValue()); } @@ -422,7 +448,7 @@ public void chineseCharacterTest() { @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertTabletWithAlignedTimeseriesTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT)); @@ -431,20 +457,20 @@ public void insertTabletWithAlignedTimeseriesTest() { long timestamp = System.currentTimeMillis(); for (long row = 0; row < 10; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue( - schemaList.get(0).getMeasurementId(), rowIndex, new SecureRandom().nextLong()); + schemaList.get(0).getMeasurementName(), rowIndex, new SecureRandom().nextLong()); tablet.addValue( - schemaList.get(1).getMeasurementId(), rowIndex, new SecureRandom().nextInt()); + schemaList.get(1).getMeasurementName(), rowIndex, new SecureRandom().nextInt()); tablet.addValue( - schemaList.get(2).getMeasurementId(), + schemaList.get(2).getMeasurementName(), rowIndex, new Binary("test", TSFileConfig.STRING_CHARSET)); timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertAlignedTablet(tablet); tablet.reset(); } @@ -464,9 +490,9 @@ public void insertTabletWithAlignedTimeseriesTest() { @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) - public void insertTabletWithNullValuesTest() { + public void insertTabletWithNullValuesTest() throws InterruptedException { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); @@ -476,30 +502,25 @@ public void insertTabletWithNullValuesTest() { Tablet tablet = new Tablet("root.sg1.d1", schemaList); for (long time = 0; time < 10; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, time); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (float) time); - tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time); - tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int) time); - tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time % 2 == 0); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (double) time); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (float) time); + tablet.addValue(schemaList.get(2).getMeasurementName(), rowIndex, time); + tablet.addValue(schemaList.get(3).getMeasurementName(), rowIndex, (int) time); + tablet.addValue(schemaList.get(4).getMeasurementName(), rowIndex, time % 2 == 0); tablet.addValue( - schemaList.get(5).getMeasurementId(), + schemaList.get(5).getMeasurementName(), rowIndex, new Binary(String.valueOf(time), TSFileConfig.STRING_CHARSET)); } - BitMap[] bitMaps = new BitMap[schemaList.size()]; for (int i = 0; i < schemaList.size(); i++) { - if (bitMaps[i] == null) { - bitMaps[i] = new BitMap(10); - } - bitMaps[i].mark(i); + tablet.getBitMaps()[i].mark(i); } - tablet.bitMaps = bitMaps; - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -512,6 +533,38 @@ public void insertTabletWithNullValuesTest() { assertEquals(9L, field.getLongV()); } } + dataSet = session.executeQueryStatement("select s3 from root.sg1.d1"); + int result = 0; + assertTrue(dataSet.hasNext()); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + Field field = rowRecord.getFields().get(0); + // skip null value + if (result == 3) { + result++; + } + assertEquals(result++, field.getIntV()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + TimeUnit.MILLISECONDS.sleep(2000); + + TestUtils.stopForciblyAndRestartDataNodes(); + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + SessionDataSet dataSet = session.executeQueryStatement("select s3 from root.sg1.d1"); + int result = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + Field field = rowRecord.getFields().get(0); + // skip null value + if (result == 3) { + result++; + } + assertEquals(result++, field.getIntV()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -522,7 +575,7 @@ public void insertTabletWithNullValuesTest() { @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertTabletWithStringValuesTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); @@ -533,22 +586,22 @@ public void insertTabletWithStringValuesTest() { Tablet tablet = new Tablet("root.sg1.d1", schemaList); for (long time = 0; time < 10; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, time); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (float) time); - tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time); - tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int) time); - tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time % 2 == 0); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (double) time); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (float) time); + tablet.addValue(schemaList.get(2).getMeasurementName(), rowIndex, time); + tablet.addValue(schemaList.get(3).getMeasurementName(), rowIndex, (int) time); + tablet.addValue(schemaList.get(4).getMeasurementName(), rowIndex, time % 2 == 0); tablet.addValue( - schemaList.get(5).getMeasurementId(), + schemaList.get(5).getMeasurementName(), rowIndex, new Binary("Text" + time, TSFileConfig.STRING_CHARSET)); - tablet.addValue(schemaList.get(6).getMeasurementId(), rowIndex, "Text" + time); + tablet.addValue(schemaList.get(6).getMeasurementName(), rowIndex, "Text" + time); } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -557,6 +610,17 @@ public void insertTabletWithStringValuesTest() { while (dataSet.hasNext()) { RowRecord rowRecord = dataSet.next(); List fields = rowRecord.getFields(); + // this test may occasionally fail by IndexOutOfBoundsException + if (fields.size() != 7) { + SessionDataSet showTimeseriesDataSet = + session.executeQueryStatement("show timeseries root.sg1.d1.*"); + LOGGER.error("show timeseries result:"); + while (showTimeseriesDataSet.hasNext()) { + RowRecord row = showTimeseriesDataSet.next(); + LOGGER.error(row.toString()); + } + LOGGER.error("The number of fields is not correct. fields values: " + fields); + } assertEquals(fields.get(5).getBinaryV(), fields.get(6).getBinaryV()); } } catch (Exception e) { @@ -569,7 +633,7 @@ public void insertTabletWithStringValuesTest() { @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertTabletWithNegativeTimestampTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); @@ -580,22 +644,22 @@ public void insertTabletWithNegativeTimestampTest() { Tablet tablet = new Tablet("root.sg1.d1", schemaList); for (long time = 0; time < 10; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, -time); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (float) time); - tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time); - tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int) time); - tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time % 2 == 0); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (double) time); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (float) time); + tablet.addValue(schemaList.get(2).getMeasurementName(), rowIndex, time); + tablet.addValue(schemaList.get(3).getMeasurementName(), rowIndex, (int) time); + tablet.addValue(schemaList.get(4).getMeasurementName(), rowIndex, time % 2 == 0); tablet.addValue( - schemaList.get(5).getMeasurementId(), + schemaList.get(5).getMeasurementName(), rowIndex, new Binary("Text" + time, TSFileConfig.STRING_CHARSET)); - tablet.addValue(schemaList.get(6).getMeasurementId(), rowIndex, "Text" + time); + tablet.addValue(schemaList.get(6).getMeasurementName(), rowIndex, "Text" + time); } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -617,7 +681,7 @@ public void insertTabletWithNegativeTimestampTest() { @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertTabletWithWrongTimestampPrecisionTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); @@ -628,22 +692,22 @@ public void insertTabletWithWrongTimestampPrecisionTest() { Tablet tablet = new Tablet("root.sg1.d1", schemaList); for (long time = 1694689856546000000L; time < 1694689856546000010L; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, time); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (float) time); - tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time); - tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int) time); - tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time % 2 == 0); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (double) time); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (float) time); + tablet.addValue(schemaList.get(2).getMeasurementName(), rowIndex, time); + tablet.addValue(schemaList.get(3).getMeasurementName(), rowIndex, (int) time); + tablet.addValue(schemaList.get(4).getMeasurementName(), rowIndex, time % 2 == 0); tablet.addValue( - schemaList.get(5).getMeasurementId(), + schemaList.get(5).getMeasurementName(), rowIndex, new Binary("Text" + time, TSFileConfig.STRING_CHARSET)); - tablet.addValue(schemaList.get(6).getMeasurementId(), rowIndex, "Text" + time); + tablet.addValue(schemaList.get(6).getMeasurementName(), rowIndex, "Text" + time); } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -656,22 +720,22 @@ public void insertTabletWithWrongTimestampPrecisionTest() { @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertTabletWithDuplicatedMeasurementsTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); Tablet tablet = new Tablet("root.sg1.d1", schemaList); for (long time = 0L; time < 10L; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, time); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (double) time); - tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, (double) time); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (double) time); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (double) time); + tablet.addValue(schemaList.get(2).getMeasurementName(), rowIndex, (double) time); } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -733,7 +797,7 @@ public void createWrongTimeSeriesTest() { LOGGER.error("", e); } - final SessionDataSet dataSet = session.executeQueryStatement("SHOW TIMESERIES"); + final SessionDataSet dataSet = session.executeQueryStatement("SHOW TIMESERIES root.sg.**"); assertFalse(dataSet.hasNext()); session.deleteStorageGroup(storageGroup); @@ -1117,6 +1181,43 @@ public void insertRecordsWithDuplicatedMeasurementsTest() { } } + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertRecordsWithExpiredDataTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + List times = new ArrayList<>(); + List> measurements = new ArrayList<>(); + List> datatypes = new ArrayList<>(); + List> values = new ArrayList<>(); + List devices = new ArrayList<>(); + + devices.add("root.sg.d1"); + addLine( + times, + measurements, + datatypes, + values, + 3L, + "s1", + "s2", + TSDataType.INT32, + TSDataType.INT32, + 1, + 2); + session.executeNonQueryStatement("set ttl to root.sg.d1 1"); + try { + session.insertRecords(devices, times, measurements, datatypes, values); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("less than ttl time bound")); + } + session.executeNonQueryStatement("unset ttl to root.sg.d1"); + SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg.d1"); + Assert.assertFalse(dataSet.hasNext()); + } + } + @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertStringRecordsOfOneDeviceWithOrderTest() { @@ -1385,11 +1486,11 @@ public void insertIllegalPathTest() { 5); long ts = 7L; for (long row = 0; row < 8; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, ts); tablet.addValue("s1", rowIndex, 1); tablet.addValue("s2", rowIndex, 1.0F); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } @@ -1424,15 +1525,15 @@ public void insertIllegalPathTest() { tablets.put("root.sg.d2", tablet2); long ts = 16L; for (long row = 0; row < 8; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); tablet1.addTimestamp(row1, ts); tablet2.addTimestamp(row2, ts); tablet1.addValue("s1", row1, 1); tablet1.addValue("s2", row1, 1.0F); tablet2.addValue("s1", row2, 1); tablet2.addValue("s2", row2, 1.0F); - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { session.insertTablets(tablets, true); tablet1.reset(); tablet2.reset(); @@ -1633,7 +1734,7 @@ public void insertPartialTablet2Test() { session.createTimeseries( "root.sg.d.s3", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); } - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT)); @@ -1643,12 +1744,12 @@ public void insertPartialTablet2Test() { long timestamp = System.currentTimeMillis(); for (long row = 0; row < 15; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue("s1", rowIndex, 1L); tablet.addValue("s2", rowIndex, 1D); tablet.addValue("s3", rowIndex, new Binary("1", TSFileConfig.STRING_CHARSET)); - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { try { session.insertTablet(tablet, true); } catch (StatementExecutionException e) { @@ -1660,7 +1761,7 @@ public void insertPartialTablet2Test() { timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { try { session.insertTablet(tablet); } catch (StatementExecutionException e) { @@ -1684,6 +1785,42 @@ public void insertPartialTablet2Test() { } } + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertPartialSQLTest() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.createAlignedTimeseries( + "root.partial_insert.d1", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.BOOLEAN), + Collections.singletonList(TSEncoding.PLAIN), + Collections.singletonList(CompressionType.UNCOMPRESSED), + null); + try { + session.executeNonQueryStatement( + "insert into root.partial_insert.d1(time, s1) values (10000,true),(20000,false),(35000,-1.5),(30000,-1),(40000,0),(50000,1),(60000,1.5),(70000,'string'),(80000,'1989-06-15'),(90000,638323200000)"); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "507: Fail to insert measurements [s1] caused by [The BOOLEAN should be true/TRUE, false/FALSE or 0/1]", + e.getMessage()); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from root.partial_insert.d1"); + long[] timestamps = new long[] {10000, 20000, 40000, 50000}; + Boolean[] values = new Boolean[] {true, false, false, true}; + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rec = dataSet.next(); + assertEquals(timestamps[cnt], rec.getTimestamp()); + assertEquals(values[cnt], rec.getFields().get(0).getBoolV()); + cnt++; + } + assertEquals(4, cnt); + } + } + @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertBinaryAsTextTest() { @@ -1709,15 +1846,15 @@ public void insertBinaryAsTextTest() { dataBinary); } // insert data using insertTablet - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.TEXT)); Tablet tablet = new Tablet("root.sg1.d1", schemaList, 100); for (int i = 0; i < bytesData.size(); i++) { byte[] data = (byte[]) bytesData.get(i); - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, i); Binary dataBinary = new Binary(data); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, dataBinary); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, dataBinary); } session.insertTablet(tablet); // check result @@ -1807,4 +1944,260 @@ public void convertRecordsToTabletsTest() { e.printStackTrace(); } } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertMinMaxTimeTest() throws IoTDBConnectionException, StatementExecutionException { + try { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"false\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + + session.executeNonQueryStatement( + String.format( + "INSERT INTO root.testInsertMinMax.d1(timestamp, s1) VALUES (%d, 1)", + Long.MIN_VALUE)); + session.executeNonQueryStatement( + String.format( + "INSERT INTO root.testInsertMinMax.d1(timestamp, s1) VALUES (%d, 1)", + Long.MAX_VALUE)); + + SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM root.testInsertMinMax.d1"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("FLUSH"); + dataSet = session.executeQueryStatement("SELECT * FROM root.testInsertMinMax.d1"); + record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + } + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"true\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + } + } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void loadMinMaxTimeNonAlignedTest() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + File file = new File("target", "test.tsfile"); + try (TsFileWriter writer = new TsFileWriter(file)) { + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.testLoadMinMax.d1"); + writer.registerTimeseries(deviceID, new MeasurementSchema("s1", TSDataType.INT32)); + TSRecord record = new TSRecord(deviceID, Long.MIN_VALUE); + record.addPoint("s1", 1); + writer.writeRecord(record); + record.setTime(Long.MAX_VALUE); + writer.writeRecord(record); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"false\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("LOAD \"" + file.getAbsolutePath() + "\""); + + SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM root.testLoadMinMax.d1"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"true\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + file.delete(); + } + } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void loadMinMaxTimeAlignedTest() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + File file = new File("target", "test.tsfile"); + try (TsFileWriter writer = new TsFileWriter(file)) { + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.testLoadMinMaxAligned.d1"); + writer.registerAlignedTimeseries( + deviceID, Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT32))); + TSRecord record = new TSRecord(deviceID, Long.MIN_VALUE); + record.addPoint("s1", 1); + writer.writeRecord(record); + record.setTime(Long.MAX_VALUE); + writer.writeRecord(record); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"false\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("LOAD \"" + file.getAbsolutePath() + "\""); + + SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM root.testLoadMinMaxAligned.d1"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"true\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + file.delete(); + } + } + + @Test + public void testWriteRestartAndDeleteDB() + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.insertRecord("root.sg1.d1", 1, Arrays.asList("s3"), Arrays.asList("1")); + + TestUtils.stopForciblyAndRestartDataNodes(); + + SessionDataSet dataSet = session.executeQueryStatement("select s3 from root.sg1.d1"); + dataSet.next(); + dataSet.close(); + + session.executeNonQueryStatement("DELETE DATABASE root.sg1"); + + session.insertRecord( + "root.sg1.d1", 1, Arrays.asList("s1", "s2", "s3"), Arrays.asList("1", "1", "1")); + + dataSet = session.executeQueryStatement("SELECT * FROM root.sg1.d1"); + RowRecord record = dataSet.next(); + assertEquals(3, record.getFields().size()); + } + } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void testQueryAllDataType() throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = + new Tablet( + "root.sg.d1", + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64), + new MeasurementSchema("s3", TSDataType.FLOAT), + new MeasurementSchema("s4", TSDataType.DOUBLE), + new MeasurementSchema("s5", TSDataType.TEXT), + new MeasurementSchema("s6", TSDataType.BOOLEAN), + new MeasurementSchema("s7", TSDataType.TIMESTAMP), + new MeasurementSchema("s8", TSDataType.BLOB), + new MeasurementSchema("s9", TSDataType.STRING), + new MeasurementSchema("s10", TSDataType.DATE), + new MeasurementSchema("s11", TSDataType.TIMESTAMP)), + 10); + tablet.addTimestamp(0, 0L); + tablet.addValue("s1", 0, 1); + tablet.addValue("s2", 0, 1L); + tablet.addValue("s3", 0, 0f); + tablet.addValue("s4", 0, 0d); + tablet.addValue("s5", 0, "text_value"); + tablet.addValue("s6", 0, true); + tablet.addValue("s7", 0, 1L); + tablet.addValue("s8", 0, new Binary(new byte[] {1})); + tablet.addValue("s9", 0, "string_value"); + tablet.addValue("s10", 0, DateUtils.parseIntToLocalDate(20250403)); + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.insertTablet(tablet); + + try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg.d1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + int count = 0; + while (iterator.next()) { + count++; + Assert.assertFalse(iterator.isNull("root.sg.d1.s1")); + Assert.assertEquals(1, iterator.getInt("root.sg.d1.s1")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s2")); + Assert.assertEquals(1L, iterator.getLong("root.sg.d1.s2")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s3")); + Assert.assertEquals(0, iterator.getFloat("root.sg.d1.s3"), 0.01); + Assert.assertFalse(iterator.isNull("root.sg.d1.s4")); + Assert.assertEquals(0, iterator.getDouble("root.sg.d1.s4"), 0.01); + Assert.assertFalse(iterator.isNull("root.sg.d1.s5")); + Assert.assertEquals("text_value", iterator.getString("root.sg.d1.s5")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s6")); + assertTrue(iterator.getBoolean("root.sg.d1.s6")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s7")); + Assert.assertEquals(new Timestamp(1), iterator.getTimestamp("root.sg.d1.s7")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s8")); + Assert.assertEquals(new Binary(new byte[] {1}), iterator.getBlob("root.sg.d1.s8")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s9")); + Assert.assertEquals("string_value", iterator.getString("root.sg.d1.s9")); + Assert.assertFalse(iterator.isNull("root.sg.d1.s10")); + Assert.assertEquals( + DateUtils.parseIntToLocalDate(20250403), iterator.getDate("root.sg.d1.s10")); + Assert.assertTrue(iterator.isNull("root.sg.d1.s11")); + Assert.assertNull(iterator.getTimestamp("root.sg.d1.s11")); + + Assert.assertEquals(new Timestamp(0), iterator.getTimestamp("Time")); + Assert.assertFalse(iterator.isNull("Time")); + } + Assert.assertEquals(tablet.getRowSize(), count); + } + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java index 1f4369f9d2e2b..080345a546747 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java @@ -33,9 +33,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -258,28 +257,20 @@ public void insertRecordsWithIllegalMeasurementTest() { @Test public void insertTabletWithIllegalMeasurementTest() { String deviceId = "root.sg1.d1"; - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("wrong`", TSDataType.INT64, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT, TSEncoding.PLAIN)); schemaList.add(new MeasurementSchema("s4", TSDataType.INT64, TSEncoding.PLAIN)); Tablet tablet = new Tablet(deviceId, schemaList, 10); - - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - for (long time = 0; time < 10; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - long[] sensor = (long[]) values[0]; - sensor[row] = time; - double[] sensor2 = (double[]) values[1]; - sensor2[row] = 0.1 + time; - Binary[] sensor3 = (Binary[]) values[2]; - sensor3[row] = BytesUtils.valueOf("ha" + time); - long[] sensor4 = (long[]) values[3]; - sensor4[row] = time; + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); + tablet.addValue(row, 0, time); + tablet.addValue(row, 1, 0.1d + time); + tablet.addValue(row, 2, "ha" + time); + tablet.addValue(row, 3, time); } try (ISession session = EnvFactory.getEnv().getSessionConnection()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java index 959771d56b734..3bc7a5984aad3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java @@ -26,15 +26,14 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -79,7 +78,7 @@ public void testInsertByStrAndSelectFailedData() { session.createTimeseries( deviceId + ".s4", TSDataType.DOUBLE, TSEncoding.RLE, CompressionType.UNCOMPRESSED); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT, TSEncoding.PLAIN)); @@ -87,20 +86,13 @@ public void testInsertByStrAndSelectFailedData() { Tablet tablet = new Tablet("root.sg1.d1", schemaList, 10); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - for (long time = 0; time < 10; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - long[] sensor = (long[]) values[0]; - sensor[row] = time; - double[] sensor2 = (double[]) values[1]; - sensor2[row] = 0.1 + time; - Binary[] sensor3 = (Binary[]) values[2]; - sensor3[row] = BytesUtils.valueOf("ha" + time); - long[] sensor4 = (long[]) values[3]; - sensor4[row] = time; + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); + tablet.addValue(row, 0, time); + tablet.addValue(row, 1, 0.1d + time); + tablet.addValue(row, 2, "ha" + time); + tablet.addValue(row, 3, time); } try { @@ -268,7 +260,7 @@ public void testInsertStrRecord() { @Test public void testInsertTablet() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); String deviceId = "root.db.d1"; schemaList.add(new MeasurementSchema("s1", TSDataType.DATE)); schemaList.add(new MeasurementSchema("s2", TSDataType.TIMESTAMP)); @@ -278,28 +270,26 @@ public void testInsertTablet() { byte[] bytes = new byte[2]; bytes[0] = (byte) Integer.parseInt("BA", 16); bytes[1] = (byte) Integer.parseInt("BE", 16); - // Method 1 to add tablet data + for (long time = 10; time < 15; time++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, time); tablet.addValue( - schemaList.get(0).getMeasurementId(), rowIndex, LocalDate.of(2024, 1, (int) time)); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, time); - tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, new Binary(bytes)); - tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, "" + time); + schemaList.get(0).getMeasurementName(), rowIndex, LocalDate.of(2024, 1, (int) time)); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, time); + tablet.addValue(schemaList.get(2).getMeasurementName(), rowIndex, new Binary(bytes)); + tablet.addValue(schemaList.get(3).getMeasurementName(), rowIndex, "" + time); } session.insertTablet(tablet); tablet.reset(); - // Method 2 to add tablet data - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; + for (long time = 15; time < 20; time++) { - int rowIndex = tablet.rowSize++; - timestamps[rowIndex] = time; - ((LocalDate[]) values[0])[rowIndex] = LocalDate.of(2024, 1, (int) time); - ((long[]) values[1])[rowIndex] = time; - ((Binary[]) values[2])[rowIndex] = new Binary(bytes); - ((Binary[]) values[3])[rowIndex] = new Binary(time + "", TSFileConfig.STRING_CHARSET); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, time); + tablet.addValue(rowIndex, 0, LocalDate.of(2024, 1, (int) time)); + tablet.addValue(rowIndex, 1, time); + tablet.addValue(rowIndex, 2, bytes); + tablet.addValue(rowIndex, 3, time + ""); } session.insertTablet(tablet); tablet.reset(); @@ -308,7 +298,7 @@ public void testInsertTablet() { Assert.assertEquals(5, columnNames.size()); for (int i = 0; i < 4; i++) { Assert.assertTrue( - columnNames.contains(deviceId + "." + schemaList.get(i).getMeasurementId())); + columnNames.contains(deviceId + "." + schemaList.get(i).getMeasurementName())); } dataSet.setFetchSize(1024); // default is 10000 int row = 10; diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java index a168dbc78ba72..7d1544a10d738 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java @@ -19,24 +19,25 @@ package org.apache.iotdb.subscription.it; -import org.apache.iotdb.session.subscription.consumer.SubscriptionExecutorServiceManager; +import org.apache.iotdb.session.subscription.consumer.base.SubscriptionExecutorServiceManager; import org.junit.After; import org.junit.Before; import org.junit.Rule; -import org.junit.rules.TestName; import org.junit.rules.TestRule; public abstract class AbstractSubscriptionIT { - @Rule public TestName testName = new TestName(); + @Rule public DisplayName testName = new DisplayName(); - @Rule public final TestRule skipOnSetUpFailure = new SkipOnSetUpFailure("setUp"); + @Rule + public final TestRule skipOnSetUpAndTearDownFailure = + new SkipOnSetUpAndTearDownFailure("setUp", "tearDown"); @Before - public void setUp() { + public void setUp() throws Exception { // set thread name - Thread.currentThread().setName(String.format("%s - main", testName.getMethodName())); + Thread.currentThread().setName(String.format("%s - main", testName.getDisplayName())); // set thread pools core size SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1); @@ -45,5 +46,5 @@ public void setUp() { } @After - public void tearDown() {} + public void tearDown() throws Exception {} } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java new file mode 100644 index 0000000000000..b1faae54287bd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it; + +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +public class DisplayName extends TestWatcher { + + private volatile String displayName; + + public DisplayName() {} + + protected void starting(final Description d) { + this.displayName = d.getDisplayName(); + } + + public String getDisplayName() { + return displayName; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java new file mode 100644 index 0000000000000..6c3891612198a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.junit.Assume; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; + +import java.lang.reflect.Method; + +public class SkipOnSetUpAndTearDownFailure implements TestRule { + + private final String setUpMethodName; + private final String tearDownMethodName; + + /** + * @param setUpMethodName Should be exactly the same as the method name decorated with @Before. + * @param tearDownMethodName Should be exactly the same as the method name decorated with @After. + */ + public SkipOnSetUpAndTearDownFailure( + @NonNull final String setUpMethodName, @NonNull final String tearDownMethodName) { + this.setUpMethodName = setUpMethodName; + this.tearDownMethodName = tearDownMethodName; + } + + @Override + public Statement apply(final Statement base, final Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + try { + base.evaluate(); + } catch (final Throwable e) { + // Pay attention to the situation of MultipleFailureException... + if ((e instanceof MultipleFailureException + && ((MultipleFailureException) e) + .getFailures().stream().allMatch(this::isExceptionInSetUpOrTearDown)) + || isExceptionInSetUpOrTearDown(e)) { + Assume.assumeTrue( + String.format( + "Skipping test due to setup or tearDown failure for %s#%s", + description.getClassName(), description.getMethodName()), + false); + } + + // Re-throw the exception (which means the test has failed). + throw e; + + // Regardless of the circumstances, the method decorated with @After will always be + // executed. + } + } + + private boolean isExceptionInSetUpOrTearDown(final Throwable e) { + // Trace back the exception stack to determine whether the exception was thrown during the + // setUp or tearDown phase. + for (final StackTraceElement stackTraceElement : e.getStackTrace()) { + if (setUpMethodName.equals(stackTraceElement.getMethodName()) + && description.getClassName().equals(stackTraceElement.getClassName()) + && isMethodAnnotationWithBefore(stackTraceElement.getMethodName())) { + e.printStackTrace(); + return true; + } + + if (tearDownMethodName.equals(stackTraceElement.getMethodName()) + && description.getClassName().equals(stackTraceElement.getClassName()) + && isMethodAnnotationWithAfter(stackTraceElement.getMethodName())) { + e.printStackTrace(); + return true; + } + } + return false; + } + + private boolean isMethodAnnotationWithBefore(final String methodName) { + try { + final Method method = description.getTestClass().getDeclaredMethod(methodName); + return method.isAnnotationPresent(org.junit.Before.class); + } catch (final Throwable ignored) { + return false; + } + } + + private boolean isMethodAnnotationWithAfter(final String methodName) { + try { + final Method method = description.getTestClass().getDeclaredMethod(methodName); + return method.isAnnotationPresent(org.junit.After.class); + } catch (final Throwable ignored) { + return false; + } + } + }; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpFailure.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpFailure.java deleted file mode 100644 index add7b7c1e2b88..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpFailure.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.subscription.it; - -import org.checkerframework.checker.nullness.qual.NonNull; -import org.junit.AssumptionViolatedException; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import java.lang.reflect.Method; - -public class SkipOnSetUpFailure implements TestRule { - - private final String setUpMethodName; - - /** - * @param setUpMethodName Should be exactly the same as the method name decorated with @Before. - */ - public SkipOnSetUpFailure(@NonNull final String setUpMethodName) { - this.setUpMethodName = setUpMethodName; - } - - @Override - public Statement apply(final Statement base, final Description description) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - try { - base.evaluate(); - } catch (final Throwable e) { - // Trace back the exception stack to determine whether the exception was thrown during the - // setUp phase. - for (final StackTraceElement stackTraceElement : e.getStackTrace()) { - if (setUpMethodName.equals(stackTraceElement.getMethodName()) - && description.getClassName().equals(stackTraceElement.getClassName()) - && isMethodAnnotationWithBefore(stackTraceElement.getMethodName())) { - e.printStackTrace(); - // Skip this test. - throw new AssumptionViolatedException( - String.format( - "Skipping test due to setup failure for %s#%s", - description.getClassName(), description.getMethodName())); - } - } - - // Re-throw the exception (which means the test has failed). - throw e; - - // Regardless of the circumstances, the method decorated with @After will always be - // executed. - } - } - - private boolean isMethodAnnotationWithBefore(final String methodName) { - try { - final Method method = description.getTestClass().getDeclaredMethod(methodName); - return method.isAnnotationPresent(org.junit.Before.class); - } catch (final Throwable ignored) { - return false; - } - } - }; - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java index 41dd654383d66..840eb895c53a1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java @@ -32,13 +32,14 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.RpcUtils; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.tsfile.utils.Pair; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -48,9 +49,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; @@ -65,7 +68,7 @@ public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); EnvFactory.getEnv() @@ -82,10 +85,10 @@ public void setUp() { @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); + + super.tearDown(); } @Test @@ -95,7 +98,7 @@ public void testSubscriptionAfterRestartCluster() throws Exception { // Create topic final String topicName = "topic1"; - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -104,10 +107,11 @@ public void testSubscriptionAfterRestartCluster() throws Exception { } // Subscription - final SubscriptionPullConsumer consumer; + final SubscriptionTreePullConsumer consumer1; + final SubscriptionTreePullConsumer consumer2; try { - consumer = - new SubscriptionPullConsumer.Builder() + consumer1 = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -116,8 +120,21 @@ public void testSubscriptionAfterRestartCluster() throws Exception { .heartbeatIntervalMs(1000) // narrow heartbeat interval .endpointsSyncIntervalMs(5000) // narrow endpoints sync interval .buildPullConsumer(); - consumer.open(); - consumer.subscribe(topicName); + consumer1.open(); + consumer1.subscribe(topicName); + + consumer2 = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c2") + .consumerGroupId("cg2") + .autoCommit(true) + .heartbeatIntervalMs(1000) // narrow heartbeat interval + .endpointsSyncIntervalMs(5000) // narrow endpoints sync interval + .buildPullConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -145,7 +162,7 @@ public void testSubscriptionAfterRestartCluster() throws Exception { client.showSubscription(new TShowSubscriptionReq()); Assert.assertEquals(RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); - Assert.assertEquals(1, showSubscriptionResp.subscriptionInfoList.size()); + Assert.assertEquals(2, showSubscriptionResp.subscriptionInfoList.size()); } // Insert some historical data @@ -162,17 +179,18 @@ public void testSubscriptionAfterRestartCluster() throws Exception { } // Subscription again - final Map timestamps = new HashMap<>(); + final Map, Long> timestamps = new ConcurrentHashMap<>(); final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = + final List threads = new ArrayList<>(); + threads.add( new Thread( () -> { - try (final SubscriptionPullConsumer consumerRef = consumer) { + try (final SubscriptionTreePullConsumer consumerRef1 = consumer1) { while (!isClosed.get()) { LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time final List messages; try { - messages = consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + messages = consumerRef1.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); } catch (final Exception e) { e.printStackTrace(); // Avoid failure @@ -183,13 +201,13 @@ public void testSubscriptionAfterRestartCluster() throws Exception { message.getSessionDataSetsHandler()) { while (dataSet.hasNext()) { final long timestamp = dataSet.next().getTimestamp(); - timestamps.put(timestamp, timestamp); + timestamps.put(new Pair<>(timestamp, consumerRef1.toString()), timestamp); } } } // Auto commit } - consumerRef.unsubscribe(topicName); + consumerRef1.unsubscribe(topicName); } catch (final Exception e) { e.printStackTrace(); // Avoid failure @@ -197,19 +215,57 @@ public void testSubscriptionAfterRestartCluster() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - %s", testName.getMethodName(), consumer)); - thread.start(); + String.format("%s - %s", testName.getDisplayName(), consumer1))); + threads.add( + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumerRef2 = consumer2) { + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages; + try { + messages = consumerRef2.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + continue; + } + for (final SubscriptionMessage message : messages) { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + final long timestamp = dataSet.next().getTimestamp(); + timestamps.put(new Pair<>(timestamp, consumerRef2.toString()), timestamp); + } + } + } + // Auto commit + } + consumerRef2.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - %s", testName.getDisplayName(), consumer2))); + for (final Thread thread : threads) { + thread.start(); + } // Check timestamps size try { // Keep retrying if there are execution failures - AWAIT.untilAsserted(() -> Assert.assertEquals(100, timestamps.size())); + AWAIT.untilAsserted(() -> Assert.assertEquals(200, timestamps.size())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); } finally { isClosed.set(true); - thread.join(); + for (final Thread thread : threads) { + thread.join(); + } } } @@ -221,7 +277,7 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { // Create topic final String topicName = "topic2"; - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -230,10 +286,10 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { } // Subscription - final SubscriptionPullConsumer consumer; + final SubscriptionTreePullConsumer consumer; try { consumer = - new SubscriptionPullConsumer.Builder() + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -280,7 +336,7 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { final Thread thread = new Thread( () -> { - try (final SubscriptionPullConsumer consumerRef = consumer) { + try (final SubscriptionTreePullConsumer consumerRef = consumer) { while (!isClosed.get()) { LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time final List messages; @@ -310,7 +366,7 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - %s", testName.getMethodName(), consumer)); + String.format("%s - %s", testName.getDisplayName(), consumer)); thread.start(); // Start DN 1 & DN 2 @@ -359,7 +415,7 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { // Create topic final String topicName = "topic3"; - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -368,10 +424,10 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { } // Subscription - final SubscriptionPullConsumer consumer; + final SubscriptionTreePullConsumer consumer; try { consumer = - new SubscriptionPullConsumer.Builder() + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -407,7 +463,7 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { final Thread thread = new Thread( () -> { - try (final SubscriptionPullConsumer consumerRef = consumer) { + try (final SubscriptionTreePullConsumer consumerRef = consumer) { while (!isClosed.get()) { LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time final List messages; @@ -437,7 +493,7 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - %s", testName.getMethodName(), consumer)); + String.format("%s - %s", testName.getDisplayName(), consumer)); thread.start(); // Shutdown leader CN diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index fb25cdc3a6c09..27d8c3c493f4a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -26,14 +26,14 @@ import org.junit.After; import org.junit.Before; -abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT { +public abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); MultiEnvFactory.createEnv(2); @@ -52,16 +52,16 @@ protected void setUpConfig() { receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); } @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { senderEnv.cleanClusterEnvironment(); receiverEnv.cleanClusterEnvironment(); + + super.tearDown(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java deleted file mode 100644 index ce8b46f2aead8..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ /dev/null @@ -1,861 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.subscription.it.dual; - -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; -import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.isession.ISession; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; -import org.apache.iotdb.rpc.RpcUtils; -import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; -import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; -import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; -import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; - -import org.apache.tsfile.read.TsFileReader; -import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.expression.QueryExpression; -import org.apache.tsfile.read.query.dataset.QueryDataSet; -import org.apache.tsfile.write.record.Tablet; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.LockSupport; - -import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) -public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT { - - private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionTopicIT.class); - - @Override - protected void setUpConfig() { - super.setUpConfig(); - - // Shorten heartbeat and sync interval to avoid timeout of snapshot mode test - senderEnv - .getConfig() - .getCommonConfig() - .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30); - senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1); - senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1); - } - - @Test - public void testTabletTopicWithPath() throws Exception { - testTopicWithPathTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); - } - - @Test - public void testTsFileTopicWithPath() throws Exception { - testTopicWithPathTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); - } - - private void testTopicWithPathTemplate(final String topicFormat) throws Exception { - // Insert some historical data on sender - try (final ISession session = senderEnv.getSessionConnection()) { - for (int i = 0; i < 100; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); - session.executeNonQueryStatement( - String.format("insert into root.db.d2(time, s) values (%s, 1)", i)); - session.executeNonQueryStatement( - String.format("insert into root.db.d3(time, t) values (%s, 1)", i)); - session.executeNonQueryStatement( - String.format("insert into root.db.t1(time, s1) values (%s, 1)", i)); - } - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Create topic on sender - final String topicName = "topic1"; - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties config = new Properties(); - config.put(TopicConstant.FORMAT_KEY, topicFormat); - config.put(TopicConstant.PATH_KEY, "root.db.*.s"); - session.createTopic(topicName, config); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - assertTopicCount(1); - - // Subscribe on sender and insert on receiver - final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = - new Thread( - () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() - .host(host) - .port(port) - .consumerId("c1") - .consumerGroupId("cg1") - .autoCommit(false) - .buildPullConsumer(); - final ISession session = receiverEnv.getSessionConnection()) { - consumer.open(); - consumer.subscribe(topicName); - while (!isClosed.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - final List messages = - consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); - insertData(messages, session); - consumer.commitSync(messages); - } - consumer.unsubscribe(topicName); - } catch (final Exception e) { - e.printStackTrace(); - // Avoid fail - } finally { - LOGGER.info("consumer exiting..."); - } - }, - String.format("%s - consumer", testName.getMethodName())); - thread.start(); - - // Check data on receiver - try { - try (final Connection connection = receiverEnv.getConnection(); - final Statement statement = connection.createStatement()) { - // Keep retrying if there are execution failures - AWAIT.untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d1.s)", "100"); - put("count(root.db.d2.s)", "100"); - } - })); - } - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } finally { - isClosed.set(true); - thread.join(); - } - } - - @Test - public void testTabletTopicWithTime() throws Exception { - testTopicWithTimeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); - } - - @Test - public void testTsFileTopicWithTime() throws Exception { - testTopicWithTimeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); - } - - private void testTopicWithTimeTemplate(final String topicFormat) throws Exception { - // Insert some historical data on sender - final long currentTime = System.currentTimeMillis(); - try (final ISession session = senderEnv.getSessionConnection()) { - for (int i = 0; i < 100; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); - session.executeNonQueryStatement( - String.format("insert into root.db.d2(time, s) values (%s, 1)", currentTime + i)); - } - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Create topic on sender - final String topicName = "topic2"; - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties config = new Properties(); - config.put(TopicConstant.FORMAT_KEY, topicFormat); - config.put(TopicConstant.START_TIME_KEY, currentTime); - session.createTopic(topicName, config); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - assertTopicCount(1); - - // Subscribe on sender and insert on receiver - final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = - new Thread( - () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() - .host(host) - .port(port) - .consumerId("c1") - .consumerGroupId("cg1") - .autoCommit(false) - .buildPullConsumer(); - final ISession session = receiverEnv.getSessionConnection()) { - consumer.open(); - consumer.subscribe(topicName); - while (!isClosed.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - final List messages = - consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); - insertData(messages, session); - consumer.commitSync(messages); - } - consumer.unsubscribe(topicName); - } catch (final Exception e) { - e.printStackTrace(); - // Avoid failure - } finally { - LOGGER.info("consumer exiting..."); - } - }, - String.format("%s - consumer", testName.getMethodName())); - thread.start(); - - // Check data on receiver - try { - try (final Connection connection = receiverEnv.getConnection(); - final Statement statement = connection.createStatement()) { - // Keep retrying if there are execution failures - AWAIT.untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d2.s)", "100"); - } - })); - } - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } finally { - isClosed.set(true); - thread.join(); - } - } - - @Test - public void testTabletTopicWithProcessor() throws Exception { - testTopicWithProcessorTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); - } - - @Test - public void testTsFileTopicWithProcessor() throws Exception { - testTopicWithProcessorTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); - } - - private void testTopicWithProcessorTemplate(final String topicFormat) throws Exception { - // Insert some history data on sender - try (final ISession session = senderEnv.getSessionConnection()) { - session.executeNonQueryStatement( - "insert into root.db.d1 (time, at1) values (1000, 1), (1500, 2), (2000, 3), (2500, 4), (3000, 5)"); - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Create topic - final String topicName = "topic3"; - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties config = new Properties(); - config.put(TopicConstant.FORMAT_KEY, topicFormat); - config.put("processor", "tumbling-time-sampling-processor"); - config.put("processor.tumbling-time.interval-seconds", "1"); - config.put("processor.down-sampling.split-file", "true"); - session.createTopic(topicName, config); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - assertTopicCount(1); - - // Subscribe on sender and insert on receiver - final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = - new Thread( - () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() - .host(host) - .port(port) - .consumerId("c1") - .consumerGroupId("cg1") - .autoCommit(false) - .buildPullConsumer(); - final ISession session = receiverEnv.getSessionConnection()) { - consumer.open(); - consumer.subscribe(topicName); - while (!isClosed.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - final List messages = - consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); - insertData(messages, session); - consumer.commitSync(messages); - } - consumer.unsubscribe(topicName); - } catch (final Exception e) { - e.printStackTrace(); - // Avoid failure - } finally { - LOGGER.info("consumer exiting..."); - } - }, - String.format("%s - consumer", testName.getMethodName())); - thread.start(); - - // Check data on receiver - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1000,1.0,"); - expectedResSet.add("2000,3.0,"); - expectedResSet.add("3000,5.0,"); - try { - try (final Connection connection = receiverEnv.getConnection(); - final Statement statement = connection.createStatement()) { - // Keep retrying if there are execution failures - AWAIT.untilAsserted( - () -> - TestUtils.assertResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select * from root.**"), - "Time,root.db.d1.at1,", - expectedResSet)); - } - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } finally { - isClosed.set(true); - thread.join(); - } - } - - @Test - public void testTopicNameWithBackQuote() throws Exception { - // Insert some historical data on sender - try (final ISession session = senderEnv.getSessionConnection()) { - for (int i = 0; i < 100; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); - } - for (int i = 100; i < 200; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); - } - for (int i = 200; i < 300; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); - } - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Create topic on sender - final String topic1 = "`topic4`"; - final String topic2 = "`'topic5'`"; - final String topic3 = "`\"topic6\"`"; - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - { - final Properties config = new Properties(); - config.put(TopicConstant.START_TIME_KEY, 0); - config.put(TopicConstant.END_TIME_KEY, 99); - session.createTopic(topic1, config); - } - { - final Properties config = new Properties(); - config.put(TopicConstant.START_TIME_KEY, 100); - config.put(TopicConstant.END_TIME_KEY, 199); - session.createTopic(topic2, config); - } - { - final Properties config = new Properties(); - config.put(TopicConstant.START_TIME_KEY, 200); - config.put(TopicConstant.END_TIME_KEY, 299); - session.createTopic(topic3, config); - } - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - assertTopicCount(3); - - // Subscribe on sender and insert on receiver - final Set topics = new HashSet<>(); - topics.add(topic1); - topics.add(topic2); - topics.add(topic3); - final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = - new Thread( - () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() - .host(host) - .port(port) - .consumerId("c1") - .consumerGroupId("cg1") - .autoCommit(false) - .buildPullConsumer(); - final ISession session = receiverEnv.getSessionConnection()) { - consumer.open(); - consumer.subscribe(topics); - while (!isClosed.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - final List messages = - consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); - for (final SubscriptionMessage message : messages) { - for (final Iterator it = - message.getSessionDataSetsHandler().tabletIterator(); - it.hasNext(); ) { - final Tablet tablet = it.next(); - session.insertTablet(tablet); - } - } - consumer.commitSync(messages); - } - consumer.unsubscribe(topics); - } catch (final Exception e) { - e.printStackTrace(); - // Avoid failure - } finally { - LOGGER.info("consumer exiting..."); - } - }, - String.format("%s - consumer", testName.getMethodName())); - thread.start(); - - // Check data on receiver - try { - try (final Connection connection = receiverEnv.getConnection(); - final Statement statement = connection.createStatement()) { - // Keep retrying if there are execution failures - AWAIT.untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), - new HashMap() { - { - put("count(root.db.d1.s)", "300"); - } - })); - } - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } finally { - isClosed.set(true); - thread.join(); - } - } - - @Test - public void testTopicWithInvalidTimeConfig() throws Exception { - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - - // Scenario 1: invalid time - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties properties = new Properties(); - properties.put(TopicConstant.START_TIME_KEY, "2024-01-32"); - properties.put(TopicConstant.END_TIME_KEY, TopicConstant.NOW_TIME_VALUE); - session.createTopic("topic7", properties); - fail(); - } catch (final Exception ignored) { - } - assertTopicCount(0); - - // Scenario 2: test when 'start-time' is greater than 'end-time' - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties properties = new Properties(); - properties.put(TopicConstant.START_TIME_KEY, "2001.01.01T08:00:00"); - properties.put(TopicConstant.END_TIME_KEY, "2000.01.01T08:00:00"); - session.createTopic("topic8", properties); - fail(); - } catch (final Exception ignored) { - } - assertTopicCount(0); - } - - @Test - public void testTabletTopicWithSnapshotMode() throws Exception { - testTopicWithSnapshotModeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); - } - - @Test - public void testTsFileTopicWithSnapshotMode() throws Exception { - testTopicWithSnapshotModeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); - } - - private void testTopicWithSnapshotModeTemplate(final String topicFormat) throws Exception { - // Insert some historical data before subscription - try (final ISession session = senderEnv.getSessionConnection()) { - for (int i = 0; i < 100; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); - } - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Create topic - final String topicName = "topic9"; - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties config = new Properties(); - config.put(TopicConstant.FORMAT_KEY, topicFormat); - config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_SNAPSHOT_VALUE); - session.createTopic(topicName, config); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - assertTopicCount(1); - - // Subscription - final AtomicInteger rowCount = new AtomicInteger(); - final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = - new Thread( - () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() - .host(host) - .port(port) - .consumerId("c1") - .consumerGroupId("cg1") - .autoCommit(false) - .buildPullConsumer()) { - consumer.open(); - consumer.subscribe(topicName); - - // Insert some data after subscription - try (final ISession session = senderEnv.getSessionConnection()) { - for (int i = 100; i < 200; ++i) { - session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); - } - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - while (!isClosed.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - final List messages = - consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); - for (final SubscriptionMessage message : messages) { - final short messageType = message.getMessageType(); - if (!SubscriptionMessageType.isValidatedMessageType(messageType)) { - LOGGER.warn("unexpected message type: {}", messageType); - continue; - } - switch (SubscriptionMessageType.valueOf(messageType)) { - case SESSION_DATA_SETS_HANDLER: - for (final SubscriptionSessionDataSet dataSet : - message.getSessionDataSetsHandler()) { - while (dataSet.hasNext()) { - dataSet.next(); - rowCount.addAndGet(1); - } - } - break; - case TS_FILE_HANDLER: - try (final TsFileReader tsFileReader = - message.getTsFileHandler().openReader()) { - final Path path = new Path("root.db.d1", "s1", true); - final QueryDataSet dataSet = - tsFileReader.query( - QueryExpression.create(Collections.singletonList(path), null)); - while (dataSet.hasNext()) { - dataSet.next(); - rowCount.addAndGet(1); - } - } - break; - default: - LOGGER.warn("unexpected message type: {}", messageType); - break; - } - } - consumer.commitSync(messages); - } - // Exiting the loop represents passing the awaitility test, at this point the result - // of 'show subscription' is empty, so there is no need to explicitly unsubscribe. - } catch (final Exception e) { - e.printStackTrace(); - // Avoid failure - } finally { - LOGGER.info("consumer exiting..."); - } - }, - String.format("%s - consumer", testName.getMethodName())); - thread.start(); - - try { - // Keep retrying if there are execution failures - AWAIT.untilAsserted( - () -> { - // Check row count - Assert.assertEquals(100, rowCount.get()); - // Check empty subscription - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TShowSubscriptionResp showSubscriptionResp = - client.showSubscription(new TShowSubscriptionReq()); - Assert.assertEquals( - RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); - Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); - Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); - } - }); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } finally { - isClosed.set(true); - thread.join(); - } - } - - @Test - public void testTabletTopicWithLooseRange() throws Exception { - testTopicWithLooseRangeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); - } - - @Test - public void testTsFileTopicWithLooseRange() throws Exception { - testTopicWithLooseRangeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); - } - - private void testTopicWithLooseRangeTemplate(final String topicFormat) throws Exception { - // Insert some historical data on sender - try (final ISession session = senderEnv.getSessionConnection()) { - session.executeNonQueryStatement( - "insert into root.db.d1 (time, at1, at2) values (1000, 1, 2), (2000, 3, 4)"); - session.executeNonQueryStatement( - "insert into root.db1.d1 (time, at1, at2) values (1000, 1, 2), (2000, 3, 4)"); - session.executeNonQueryStatement( - "insert into root.db.d1 (time, at1, at2) values (3000, 1, 2), (4000, 3, 4)"); - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Create topic - final String topicName = "topic10"; - final String host = senderEnv.getIP(); - final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { - session.open(); - final Properties config = new Properties(); - config.put(TopicConstant.FORMAT_KEY, topicFormat); - config.put(TopicConstant.LOOSE_RANGE_KEY, TopicConstant.LOOSE_RANGE_ALL_VALUE); - config.put(TopicConstant.PATH_KEY, "root.db.d1.at1"); - config.put(TopicConstant.START_TIME_KEY, "1500"); - config.put(TopicConstant.END_TIME_KEY, "2500"); - session.createTopic(topicName, config); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - assertTopicCount(1); - - final AtomicBoolean dataPrepared = new AtomicBoolean(false); - final AtomicBoolean topicSubscribed = new AtomicBoolean(false); - final AtomicBoolean finished = new AtomicBoolean(false); - final List threads = new ArrayList<>(); - - // Subscribe on sender - threads.add( - new Thread( - () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() - .host(host) - .port(port) - .consumerId("c1") - .consumerGroupId("cg1") - .autoCommit(false) - .buildPullConsumer(); - final ISession session = receiverEnv.getSessionConnection()) { - consumer.open(); - consumer.subscribe(topicName); - topicSubscribed.set(true); - while (!finished.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - if (dataPrepared.get()) { - final List messages = - consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); - insertData(messages, session); - consumer.commitSync(messages); - } - } - consumer.unsubscribe(topicName); - } catch (final Exception e) { - e.printStackTrace(); - // Avoid failure - } finally { - LOGGER.info("consumer exiting..."); - } - }, - String.format("%s - consumer", testName.getMethodName()))); - - // Insert some realtime data on sender - threads.add( - new Thread( - () -> { - while (!topicSubscribed.get()) { - LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time - } - try (final ISession session = senderEnv.getSessionConnection()) { - session.executeNonQueryStatement( - "insert into root.db.d1 (time, at1, at2) values (1001, 1, 2), (2001, 3, 4)"); - session.executeNonQueryStatement( - "insert into root.db1.d1 (time, at1, at2) values (1001, 1, 2), (2001, 3, 4)"); - session.executeNonQueryStatement( - "insert into root.db.d1 (time, at1, at2) values (3001, 1, 2), (4001, 3, 4)"); - session.executeNonQueryStatement("flush"); - } catch (final Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - dataPrepared.set(true); - }, - String.format("%s - data inserter", testName.getMethodName()))); - - for (final Thread thread : threads) { - thread.start(); - } - - try (final Connection connection = receiverEnv.getConnection(); - final Statement statement = connection.createStatement()) { - // Keep retrying if there are execution failures - AWAIT.untilAsserted( - () -> - TestUtils.assertSingleResultSetEqual( - TestUtils.executeQueryWithRetry( - statement, - "select count(at1) from root.db.d1 where time >= 1500 and time <= 2500"), - new HashMap() { - { - put("count(root.db.d1.at1)", "2"); - } - })); - } - - finished.set(true); - for (final Thread thread : threads) { - thread.join(); - } - } - - /////////////////////////////// utility /////////////////////////////// - - private void assertTopicCount(final int count) throws Exception { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showTopicResult = - client.showTopic(new TShowTopicReq()).topicInfoList; - Assert.assertEquals(count, showTopicResult.size()); - } - } - - private void insertData(final List messages, final ISession session) - throws Exception { - for (final SubscriptionMessage message : messages) { - final short messageType = message.getMessageType(); - if (!SubscriptionMessageType.isValidatedMessageType(messageType)) { - LOGGER.warn("unexpected message type: {}", messageType); - continue; - } - switch (SubscriptionMessageType.valueOf(messageType)) { - case SESSION_DATA_SETS_HANDLER: - for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); - it.hasNext(); ) { - final Tablet tablet = it.next(); - session.insertTablet(tablet); - } - break; - case TS_FILE_HANDLER: - final SubscriptionTsFileHandler tsFileHandler = message.getTsFileHandler(); - session.executeNonQueryStatement( - String.format("load '%s'", tsFileHandler.getFile().getAbsolutePath())); - break; - default: - LOGGER.warn("unexpected message type: {}", messageType); - break; - } - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionTopicIT.java new file mode 100644 index 0000000000000..d618bb95138dc --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionTopicIT.java @@ -0,0 +1,407 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.dual.tablemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableArchVerification; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.iotdb.subscription.it.dual.AbstractSubscriptionDualIT; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; +import java.util.function.Consumer; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTableArchVerification.class}) +public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionTopicIT.class); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void setUpConfig() { + super.setUpConfig(); + + // Shorten heartbeat and sync interval to avoid timeout of snapshot mode test + senderEnv + .getConfig() + .getCommonConfig() + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30); + senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1); + senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1); + } + + @Test + public void testTabletTopicWithPath() throws Exception { + testTopicWithPathTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithPath() throws Exception { + testTopicWithPathTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithPathTemplate(final String topicFormat) throws Exception { + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test2"); + TableModelUtils.createDataBaseAndTable(senderEnv, "foo", "foo"); + + TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(receiverEnv, "test2", "test2"); + TableModelUtils.createDataBaseAndTable(receiverEnv, "foo", "foo"); + + // Insert some historical data on sender + TableModelUtils.insertData("test1", "test1", 0, 10, senderEnv); + TableModelUtils.insertData("test2", "test2", 0, 10, senderEnv); + TableModelUtils.insertData("foo", "foo", 0, 10, senderEnv); + + // Create topic on sender + final String topicName = "topic1"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.DATABASE_KEY, "test.*"); + config.put(TopicConstant.TABLE_KEY, "test.*"); + session.createTopic(topicName, config); + } + assertTopicCount(1); + + // Subscribe on sender and insert on receiver + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final ISubscriptionTablePullConsumer consumer = + new SubscriptionTablePullConsumerBuilder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .build(); + final ITableSession session = receiverEnv.getTableSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid fail + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Check data on receiver + try { + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> { + TableModelUtils.assertData("test1", "test1", 0, 10, receiverEnv, handleFailure); + TableModelUtils.assertData("test2", "test2", 0, 10, receiverEnv, handleFailure); + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTabletTopicWithTime() throws Exception { + testTopicWithTimeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithTime() throws Exception { + testTopicWithTimeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithTimeTemplate(final String topicFormat) throws Exception { + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); + + // Insert some historical data on sender + TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + + // Create topic on sender + final String topicName = "topic2"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.START_TIME_KEY, 25); + config.put(TopicConstant.END_TIME_KEY, 75); + session.createTopic(topicName, config); + } + assertTopicCount(1); + + // Subscribe on sender and insert on receiver + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final ISubscriptionTablePullConsumer consumer = + new SubscriptionTablePullConsumerBuilder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .build(); + final ITableSession session = receiverEnv.getTableSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid fail + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Check data on receiver + try { + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> TableModelUtils.assertData("test1", "test1", 25, 76, receiverEnv, handleFailure)); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTabletTopicWithSnapshotMode() throws Exception { + testTopicWithSnapshotModeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithSnapshotMode() throws Exception { + testTopicWithSnapshotModeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithSnapshotModeTemplate(final String topicFormat) throws Exception { + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); + + // Insert some historical data on sender + TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + + // Create topic + final String topicName = "topic3"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_SNAPSHOT_VALUE); + session.createTopic(topicName, config); + } + assertTopicCount(1); + + // Subscription + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final ISubscriptionTablePullConsumer consumer = + new SubscriptionTablePullConsumerBuilder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .build(); + final ITableSession session = receiverEnv.getTableSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + + // Insert some realtime data on sender + TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + + // Exiting the loop represents passing the awaitility test, at this point the result + // of 'show subscription' is empty, so there is no need to explicitly unsubscribe. + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + try { + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> { + // Check empty subscription + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TShowSubscriptionResp showSubscriptionResp = + client.showSubscription(new TShowSubscriptionReq()); + Assert.assertEquals( + RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); + Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); + Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); + } + // Check data + TableModelUtils.assertData("test1", "test1", 0, 100, receiverEnv, handleFailure); + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + /////////////////////////////// utility /////////////////////////////// + + private void assertTopicCount(final int count) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showTopicResult = + client.showTopic(new TShowTopicReq().setIsTableModel(true)).topicInfoList; + Assert.assertEquals(count, showTopicResult.size()); + } + } + + private void insertData(final List messages, final ITableSession session) + throws Exception { + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (!SubscriptionMessageType.isValidatedMessageType(messageType)) { + LOGGER.warn("unexpected message type: {}", messageType); + continue; + } + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : message.getSessionDataSetsHandler()) { + session.executeNonQueryStatement( + "use " + Objects.requireNonNull(dataSet.getDatabaseName())); + session.insert(dataSet.getTablet()); + } + break; + case TS_FILE_HANDLER: + final SubscriptionTsFileHandler tsFileHandler = message.getTsFileHandler(); + session.executeNonQueryStatement( + "use " + Objects.requireNonNull(tsFileHandler.getDatabaseName())); + session.executeNonQueryStatement( + String.format("load '%s'", tsFileHandler.getFile().getAbsolutePath())); + break; + default: + LOGGER.warn("unexpected message type: {}", messageType); + break; + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java rename to integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java index eccfc634e6b9e..ece07a991616c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.subscription.it.dual; +package org.apache.iotdb.subscription.it.dual.treemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,15 +26,16 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeArchVerification; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.iotdb.subscription.it.dual.AbstractSubscriptionDualIT; import org.apache.tsfile.read.TsFileReader; import org.apache.tsfile.read.common.Path; @@ -69,7 +70,7 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionTreeArchVerification.class}) public class IoTDBSubscriptionConsumerGroupIT extends AbstractSubscriptionDualIT { // Test dimensions: @@ -123,7 +124,7 @@ protected void setUpConfig() { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); // Setup connector attributes @@ -836,7 +837,7 @@ private void createTopics(final long currentTime) { // Create topics on sender final String host = senderEnv.getIP(); final int port = Integer.parseInt(senderEnv.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); { final Properties config = new Properties(); @@ -939,10 +940,10 @@ private void createPipes(final long currentTime, final Map conne } } - private SubscriptionPullConsumer createConsumerAndSubscribeTopics( + private SubscriptionTreePullConsumer createConsumerAndSubscribeTopics( final SubscriptionInfo subscriptionInfo) { - final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(senderEnv.getIP()) .port(Integer.parseInt(senderEnv.getPort())) .consumerId(subscriptionInfo.consumerId) @@ -956,7 +957,7 @@ private SubscriptionPullConsumer createConsumerAndSubscribeTopics( } private void pollMessagesAndCheck( - final List consumers, + final List consumers, final Map expectedHeaderWithResult) throws Exception { final AtomicBoolean isClosed = new AtomicBoolean(false); @@ -969,7 +970,7 @@ private void pollMessagesAndCheck( final Thread t = new Thread( () -> { - try (final SubscriptionPullConsumer consumer = consumers.get(index)) { + try (final SubscriptionTreePullConsumer consumer = consumers.get(index)) { while (!isClosed.get()) { LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time final List messages = @@ -1036,7 +1037,7 @@ private void pollMessagesAndCheck( LOGGER.info("consumer {} exiting...", consumers.get(index)); } }, - String.format("%s - %s", testName.getMethodName(), consumers.get(index).toString())); + String.format("%s - %s", testName.getDisplayName(), consumers.get(index).toString())); t.start(); threads.add(t); } @@ -1058,7 +1059,7 @@ private void pollMessagesAndCheck( for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { // wrapper.executeJstack(); wrapper.executeJstack( - String.format("%s_%s", testName.getMethodName(), currentTime[0])); + String.format("%s_%s", testName.getDisplayName(), currentTime[0])); } currentTime[0] = System.currentTimeMillis(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java similarity index 89% rename from integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java rename to integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java index 39a9f2225f6bb..f24b832153f7b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java @@ -17,19 +17,21 @@ * under the License. */ -package org.apache.iotdb.subscription.it.dual; +package org.apache.iotdb.subscription.it.dual.treemodel; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeArchVerification; import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.iotdb.subscription.it.dual.AbstractSubscriptionDualIT; import org.apache.tsfile.write.record.Tablet; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -49,12 +51,18 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionTreeArchVerification.class}) public class IoTDBSubscriptionTimePrecisionIT extends AbstractSubscriptionDualIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionTimePrecisionIT.class); + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + @Override protected void setUpConfig() { super.setUpConfig(); @@ -87,7 +95,7 @@ public void testTopicTimePrecision() throws Exception { // Create topic on sender final String topic1 = "topic1"; final String topic2 = "topic2"; - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); { final Properties config = new Properties(); @@ -127,8 +135,8 @@ public void testTopicTimePrecision() throws Exception { final Thread thread = new Thread( () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -160,7 +168,7 @@ public void testTopicTimePrecision() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check data on receiver diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java new file mode 100644 index 0000000000000..ccce4d9f0d296 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java @@ -0,0 +1,941 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.dual.treemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeArchVerification; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.iotdb.subscription.it.dual.AbstractSubscriptionDualIT; + +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeArchVerification.class}) +public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionTopicIT.class); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void setUpConfig() { + super.setUpConfig(); + + // Shorten heartbeat and sync interval to avoid timeout of snapshot mode test + senderEnv + .getConfig() + .getCommonConfig() + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30); + senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1); + senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1); + } + + @Test + public void testTabletTopicWithPath() throws Exception { + testTopicWithPathTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithPath() throws Exception { + testTopicWithPathTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithPathTemplate(final String topicFormat) throws Exception { + // Insert some historical data on sender + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); + session.executeNonQueryStatement( + String.format("insert into root.db.d2(time, s) values (%s, 1)", i)); + session.executeNonQueryStatement( + String.format("insert into root.db.d3(time, t) values (%s, 1)", i)); + session.executeNonQueryStatement( + String.format("insert into root.db.t1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic on sender + final String topicName = "topic1"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.PATH_KEY, "root.db.*.s"); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(1); + + // Subscribe on sender and insert on receiver + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer(); + final ISession session = receiverEnv.getSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid fail + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Check data on receiver + try { + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d1.s)", "100"); + put("count(root.db.d2.s)", "100"); + } + })); + } + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTabletTopicWithTime() throws Exception { + testTopicWithTimeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithTime() throws Exception { + testTopicWithTimeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithTimeTemplate(final String topicFormat) throws Exception { + // Insert some historical data on sender + final long currentTime = System.currentTimeMillis(); + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); + session.executeNonQueryStatement( + String.format("insert into root.db.d2(time, s) values (%s, 1)", currentTime + i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic on sender + final String topicName = "topic2"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.START_TIME_KEY, currentTime); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(1); + + // Subscribe on sender and insert on receiver + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer(); + final ISession session = receiverEnv.getSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Check data on receiver + try { + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d2.s)", "100"); + } + })); + } + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTabletTopicWithProcessor() throws Exception { + testTopicWithProcessorTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithProcessor() throws Exception { + testTopicWithProcessorTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithProcessorTemplate(final String topicFormat) throws Exception { + // Insert some history data on sender + try (final ISession session = senderEnv.getSessionConnection()) { + session.executeNonQueryStatement( + "insert into root.db.d1 (time, at1) values (1000, 1), (1500, 2), (2000, 3), (2500, 4), (3000, 5)"); + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic + final String topicName = "topic3"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put("processor", "tumbling-time-sampling-processor"); + config.put("processor.tumbling-time.interval-seconds", "1"); + config.put("processor.down-sampling.split-file", "true"); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(1); + + // Subscribe on sender and insert on receiver + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer(); + final ISession session = receiverEnv.getSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Check data on receiver + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1000,1.0,"); + expectedResSet.add("2000,3.0,"); + expectedResSet.add("3000,5.0,"); + try { + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> + TestUtils.assertResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select * from root.**"), + "Time,root.db.d1.at1,", + expectedResSet)); + } + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTopicNameWithBackQuote() throws Exception { + // Insert some historical data on sender + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); + } + for (int i = 100; i < 200; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); + } + for (int i = 200; i < 300; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic on sender + final String topic1 = "`topic4`"; + final String topic2 = "`'topic5'`"; + final String topic3 = "`\"topic6\"`"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + { + final Properties config = new Properties(); + config.put(TopicConstant.START_TIME_KEY, 0); + config.put(TopicConstant.END_TIME_KEY, 99); + session.createTopic(topic1, config); + } + { + final Properties config = new Properties(); + config.put(TopicConstant.START_TIME_KEY, 100); + config.put(TopicConstant.END_TIME_KEY, 199); + session.createTopic(topic2, config); + } + { + final Properties config = new Properties(); + config.put(TopicConstant.START_TIME_KEY, 200); + config.put(TopicConstant.END_TIME_KEY, 299); + session.createTopic(topic3, config); + } + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(3); + + // Subscribe on sender and insert on receiver + final Set topics = new HashSet<>(); + topics.add(topic1); + topics.add(topic2); + topics.add(topic3); + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer(); + final ISession session = receiverEnv.getSessionConnection()) { + consumer.open(); + consumer.subscribe(topics); + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + for (final SubscriptionMessage message : messages) { + for (final Iterator it = + message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + } + } + consumer.commitSync(messages); + } + consumer.unsubscribe(topics); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Check data on receiver + try { + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), + new HashMap() { + { + put("count(root.db.d1.s)", "300"); + } + })); + } + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTopicWithInvalidTimeConfig() throws Exception { + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + + // Scenario 1: invalid time + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "2024-01-32"); + properties.put(TopicConstant.END_TIME_KEY, TopicConstant.NOW_TIME_VALUE); + session.createTopic("topic7", properties); + fail(); + } catch (final Exception ignored) { + } + assertTopicCount(0); + + // Scenario 2: test when 'start-time' is greater than 'end-time' + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "2001.01.01T08:00:00"); + properties.put(TopicConstant.END_TIME_KEY, "2000.01.01T08:00:00"); + session.createTopic("topic8", properties); + fail(); + } catch (final Exception ignored) { + } + assertTopicCount(0); + } + + @Test + public void testTabletTopicWithSnapshotMode() throws Exception { + testTopicWithSnapshotModeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithSnapshotMode() throws Exception { + testTopicWithSnapshotModeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithSnapshotModeTemplate(final String topicFormat) throws Exception { + // Insert some historical data before subscription + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic + final String topicName = "topic9"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_SNAPSHOT_VALUE); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(1); + + // Subscription + final AtomicInteger rowCount = new AtomicInteger(); + final AtomicBoolean isClosed = new AtomicBoolean(false); + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + + // Insert some data after subscription + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 100; i < 200; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (!SubscriptionMessageType.isValidatedMessageType(messageType)) { + LOGGER.warn("unexpected message type: {}", messageType); + continue; + } + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + break; + case TS_FILE_HANDLER: + try (final TsFileReader tsFileReader = + message.getTsFileHandler().openReader()) { + final Path path = new Path("root.db.d1", "s1", true); + final QueryDataSet dataSet = + tsFileReader.query( + QueryExpression.create(Collections.singletonList(path), null)); + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + break; + default: + LOGGER.warn("unexpected message type: {}", messageType); + break; + } + } + consumer.commitSync(messages); + } + // Exiting the loop represents passing the awaitility test, at this point the result + // of 'show subscription' is empty, so there is no need to explicitly unsubscribe. + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + try { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> { + // Check row count + Assert.assertEquals(100, rowCount.get()); + // Check empty subscription + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TShowSubscriptionResp showSubscriptionResp = + client.showSubscription(new TShowSubscriptionReq()); + Assert.assertEquals( + RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); + Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); + Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); + } + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + isClosed.set(true); + thread.join(); + } + } + + @Test + public void testTabletTopicWithLooseRange() throws Exception { + testTopicWithLooseRangeTemplate(TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + } + + @Test + public void testTsFileTopicWithLooseRange() throws Exception { + testTopicWithLooseRangeTemplate(TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); + } + + private void testTopicWithLooseRangeTemplate(final String topicFormat) throws Exception { + // Insert some historical data on sender + try (final ISession session = senderEnv.getSessionConnection()) { + session.executeNonQueryStatement( + "insert into root.db.d1 (time, at1, at2) values (1000, 1, 2), (2000, 3, 4)"); + session.executeNonQueryStatement( + "insert into root.db1.d1 (time, at1, at2) values (1000, 1, 2), (2000, 3, 4)"); + session.executeNonQueryStatement( + "insert into root.db.d1 (time, at1, at2) values (3000, 1, 2), (4000, 3, 4)"); + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic + final String topicName = "topic10"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.FORMAT_KEY, topicFormat); + config.put(TopicConstant.LOOSE_RANGE_KEY, TopicConstant.LOOSE_RANGE_ALL_VALUE); + config.put(TopicConstant.PATH_KEY, "root.db.d1.at1"); + config.put(TopicConstant.START_TIME_KEY, "1500"); + config.put(TopicConstant.END_TIME_KEY, "2500"); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(1); + + final AtomicBoolean dataPrepared = new AtomicBoolean(false); + final AtomicBoolean topicSubscribed = new AtomicBoolean(false); + final AtomicBoolean finished = new AtomicBoolean(false); + final List threads = new ArrayList<>(); + + // Subscribe on sender + threads.add( + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(false) + .buildPullConsumer(); + final ISession session = receiverEnv.getSessionConnection()) { + consumer.open(); + consumer.subscribe(topicName); + topicSubscribed.set(true); + while (!finished.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + if (dataPrepared.get()) { + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + insertData(messages, session); + consumer.commitSync(messages); + } + } + consumer.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName()))); + + // Insert some realtime data on sender + threads.add( + new Thread( + () -> { + while (!topicSubscribed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + } + try (final ISession session = senderEnv.getSessionConnection()) { + session.executeNonQueryStatement( + "insert into root.db.d1 (time, at1, at2) values (1001, 1, 2), (2001, 3, 4)"); + session.executeNonQueryStatement( + "insert into root.db1.d1 (time, at1, at2) values (1001, 1, 2), (2001, 3, 4)"); + session.executeNonQueryStatement( + "insert into root.db.d1 (time, at1, at2) values (3001, 1, 2), (4001, 3, 4)"); + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + dataPrepared.set(true); + }, + String.format("%s - data inserter", testName.getDisplayName()))); + + for (final Thread thread : threads) { + thread.start(); + } + + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> + TestUtils.assertSingleResultSetEqual( + TestUtils.executeQueryWithRetry( + statement, + "select count(at1) from root.db.d1 where time >= 1500 and time <= 2500"), + new HashMap() { + { + put("count(root.db.d1.at1)", "2"); + } + })); + } + + finished.set(true); + for (final Thread thread : threads) { + thread.join(); + } + } + + @Test + public void testSnapshotModeWithEmptyData() throws Exception { + // Create topic + final String topicName = "topic11"; + final String host = senderEnv.getIP(); + final int port = Integer.parseInt(senderEnv.getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_SNAPSHOT_VALUE); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertTopicCount(1); + + // Subscription + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(true) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + + while (!consumer.allTopicMessagesHaveBeenConsumed()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); // poll and ignore + } + + // Exiting the loop represents passing the awaitility test, at this point the result + // of 'show subscription' is empty, so there is no need to explicitly unsubscribe. + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + try { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> { + // Check empty subscription + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TShowSubscriptionResp showSubscriptionResp = + client.showSubscription(new TShowSubscriptionReq()); + Assert.assertEquals( + RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); + Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); + Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); + } + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + thread.join(); + } + } + + /////////////////////////////// utility /////////////////////////////// + + private void assertTopicCount(final int count) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showTopicResult = + client.showTopic(new TShowTopicReq()).topicInfoList; + Assert.assertEquals(count, showTopicResult.size()); + } + } + + private void insertData(final List messages, final ISession session) + throws Exception { + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (!SubscriptionMessageType.isValidatedMessageType(messageType)) { + LOGGER.warn("unexpected message type: {}", messageType); + continue; + } + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + } + break; + case TS_FILE_HANDLER: + final SubscriptionTsFileHandler tsFileHandler = message.getTsFileHandler(); + session.executeNonQueryStatement( + String.format("load '%s'", tsFileHandler.getFile().getAbsolutePath())); + break; + default: + LOGGER.warn("unexpected message type: {}", messageType); + break; + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java index bb248f745d4ed..5fa7c5808fa86 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java @@ -25,11 +25,11 @@ import org.junit.After; import org.junit.Before; -abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT { +public abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); EnvFactory.getEnv().initClusterEnvironment(); @@ -37,9 +37,9 @@ public void setUp() { @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); + + super.tearDown(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 68287ab66193e..eab487a0b958d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -19,17 +19,22 @@ package org.apache.iotdb.subscription.it.local; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.session.subscription.SubscriptionSession; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; import org.apache.iotdb.session.subscription.consumer.AckStrategy; import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback; import org.apache.iotdb.session.subscription.consumer.ConsumeResult; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.model.Subscription; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; @@ -39,6 +44,7 @@ import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -50,6 +56,7 @@ import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -65,6 +72,12 @@ public class IoTDBSubscriptionBasicIT extends AbstractSubscriptionLocalIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionBasicIT.class); + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + @Test public void testBasicPullConsumerWithCommitAsync() throws Exception { // Insert some historical data @@ -83,7 +96,7 @@ public void testBasicPullConsumerWithCommitAsync() throws Exception { final String topicName = "topic1"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -100,8 +113,8 @@ public void testBasicPullConsumerWithCommitAsync() throws Exception { final Thread thread = new Thread( () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -162,7 +175,7 @@ public void onFailure(final Throwable e) { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check row count @@ -234,7 +247,7 @@ public void testBasicPushConsumer() { final String topicName = "topic2"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -243,8 +256,8 @@ public void testBasicPushConsumer() { } // Subscription - try (final SubscriptionPushConsumer consumer = - new SubscriptionPushConsumer.Builder() + try (final SubscriptionTreePushConsumer consumer = + new SubscriptionTreePushConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -256,7 +269,7 @@ public void testBasicPushConsumer() { message .getSessionDataSetsHandler() .tabletIterator() - .forEachRemaining(tablet -> rowCount.addAndGet(tablet.rowSize)); + .forEachRemaining(tablet -> rowCount.addAndGet(tablet.getRowSize())); return ConsumeResult.SUCCESS; }) .buildPushConsumer()) { @@ -334,7 +347,7 @@ public void testPollUnsubscribedTopics() throws Exception { final String topicName4 = "topic4"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); { final Properties properties = new Properties(); @@ -358,8 +371,8 @@ public void testPollUnsubscribedTopics() throws Exception { final Thread thread = new Thread( () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -391,7 +404,7 @@ public void testPollUnsubscribedTopics() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check row count @@ -429,7 +442,7 @@ public void testTsFileDeduplication() { final String topicName = "topic5"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); final Properties config = new Properties(); config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE); @@ -442,8 +455,8 @@ public void testTsFileDeduplication() { // Subscription final AtomicInteger onReceiveCount = new AtomicInteger(); final AtomicInteger rowCount = new AtomicInteger(); - try (final SubscriptionPushConsumer consumer = - new SubscriptionPushConsumer.Builder() + try (final SubscriptionTreePushConsumer consumer = + new SubscriptionTreePushConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -504,7 +517,7 @@ public void testDataSetDeduplication() { final String topicName = "topic6"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); final Properties config = new Properties(); config.put(TopicConstant.PATTERN_KEY, "root.db.d1.s1"); @@ -516,8 +529,8 @@ public void testDataSetDeduplication() { // Subscription final AtomicInteger rowCount = new AtomicInteger(); - try (final SubscriptionPushConsumer consumer = - new SubscriptionPushConsumer.Builder() + try (final SubscriptionTreePushConsumer consumer = + new SubscriptionTreePushConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -545,4 +558,164 @@ public void testDataSetDeduplication() { fail(e.getMessage()); } } + + // same to + // org.apache.iotdb.subscription.it.local.IoTDBSubscriptionBasicIT.testDataSetDeduplication, + // but missing consumer id & consumer group id when building consumer + @Test + public void testMissingConsumerId() { + // Insert some historical data + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.createDatabase("root.db"); + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1, s2) values (%s, 1, 2)", i)); + session.executeNonQueryStatement( + String.format("insert into root.db.d2(time, s1, s2) values (%s, 3, 4)", i)); + } + // DO NOT FLUSH HERE + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic + final String topicName = "topic7"; + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties config = new Properties(); + config.put(TopicConstant.PATTERN_KEY, "root.db.d1.s1"); + session.createTopic(topicName, config); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Subscription + final AtomicInteger rowCount = new AtomicInteger(); + try (final SubscriptionTreePushConsumer consumer = + new SubscriptionTreePushConsumer.Builder() + .host(host) + .port(port) + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()) { + + consumer.open(); + consumer.subscribe(topicName); + + AWAIT.untilAsserted( + () -> { + Assert.assertEquals(100, rowCount.get()); + Assert.assertNotNull(consumer.getConsumerId()); + Assert.assertNotNull(consumer.getConsumerGroupId()); + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testDropSubscriptionBySession() throws Exception { + // Insert some historical data + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (int i = 0; i < 100; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create topic + final String topicName = "topic8"; + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + session.createTopic(topicName); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Subscription + final Thread thread = + new Thread( + () -> { + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c1") + .consumerGroupId("cg1") + .autoCommit(true) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + + while (!consumer.allTopicMessagesHaveBeenConsumed()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); // poll and ignore + } + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - consumer", testName.getDisplayName())); + thread.start(); + + // Drop Subscription + LockSupport.parkNanos(5_000_000_000L); // wait some time + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Set subscriptions = session.getSubscriptions(topicName); + Assert.assertEquals(1, subscriptions.size()); + session.dropSubscription(subscriptions.iterator().next().getSubscriptionId()); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try { + // Keep retrying if there are execution failures + AWAIT.untilAsserted( + () -> { + // Check empty subscription + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) + EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final TShowSubscriptionResp showSubscriptionResp = + client.showSubscription(new TShowSubscriptionReq()); + Assert.assertEquals( + RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); + Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); + Assert.assertEquals(0, showSubscriptionResp.subscriptionInfoList.size()); + } + }); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + thread.join(); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java index 413540a89c1f1..7b45081797bce 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java @@ -24,8 +24,8 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; @@ -41,6 +41,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -64,6 +65,12 @@ public class IoTDBSubscriptionDataTypeIT extends AbstractSubscriptionLocalIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionDataTypeIT.class); + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + // ----------------------------- // // SessionDataSetsHandler format // // ----------------------------- // @@ -242,7 +249,7 @@ private void testPullConsumerSubscribeDataTemplate( final String topicName = "topic1"; final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); final Properties config = new Properties(); config.put(TopicConstant.FORMAT_KEY, topicFormat); @@ -258,8 +265,8 @@ private void testPullConsumerSubscribeDataTemplate( final Thread thread = new Thread( () -> { - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -326,7 +333,7 @@ private void testPullConsumerSubscribeDataTemplate( LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check row count @@ -345,22 +352,16 @@ private void testPullConsumerSubscribeDataTemplate( private Object getValue(final TSDataType type, final Tablet tablet) { switch (type) { case BOOLEAN: - return ((boolean[]) tablet.values[0])[0]; case INT32: - return ((int[]) tablet.values[0])[0]; case INT64: case TIMESTAMP: - return ((long[]) tablet.values[0])[0]; case FLOAT: - return ((float[]) tablet.values[0])[0]; case DOUBLE: - return ((double[]) tablet.values[0])[0]; case TEXT: case BLOB: case STRING: - return ((Binary[]) tablet.values[0])[0]; case DATE: - return ((LocalDate[]) tablet.values[0])[0]; + return tablet.getValue(0, 0); default: return null; } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java index 85fff437f18e6..d9f74d0966c49 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionIdempotentIT.java @@ -22,10 +22,11 @@ import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -40,6 +41,12 @@ public class IoTDBSubscriptionIdempotentIT extends AbstractSubscriptionLocalIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionIdempotentIT.class); + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + @Test public void testSubscribeOrUnsubscribeNonExistedTopicTest() { final String host = EnvFactory.getEnv().getIP(); @@ -47,8 +54,8 @@ public void testSubscribeOrUnsubscribeNonExistedTopicTest() { // Subscribe non-existed topic final String topicName = "topic1"; - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -64,8 +71,8 @@ public void testSubscribeOrUnsubscribeNonExistedTopicTest() { } // Unsubscribe non-existed topic - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -88,7 +95,7 @@ public void testSubscribeExistedSubscribedTopicTest() { // Create topic final String topicName = "topic2"; - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -96,8 +103,8 @@ public void testSubscribeExistedSubscribedTopicTest() { Assert.fail(e.getMessage()); } - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") @@ -123,7 +130,7 @@ public void testUnsubscribeExistedNonSubscribedTopicTest() { // Create topic final String topicName = "topic3"; - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); session.createTopic(topicName); } catch (final Exception e) { @@ -131,8 +138,8 @@ public void testUnsubscribeExistedNonSubscribedTopicTest() { Assert.fail(e.getMessage()); } - try (final SubscriptionPullConsumer consumer = - new SubscriptionPullConsumer.Builder() + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() .host(host) .port(port) .consumerId("c1") diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionTopicIT.java new file mode 100644 index 0000000000000..15a5e52e04f35 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionTopicIT.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.local; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.model.Topic; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionLocalIT { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void testBasicCreateTopic() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + // create topic + String topicName = "topic1"; + session.createTopic(topicName); + Assert.assertTrue(session.getTopic(topicName).isPresent()); + Assert.assertEquals(topicName, session.getTopic(topicName).get().getTopicName()); + + // create topic + topicName = "topic2"; + Properties properties = new Properties(); + properties.put("path", "root.**"); + properties.put("start-time", "2023-01-01"); + properties.put("end-time", "2023-12-31"); + properties.put("format", "TsFileHandler"); + session.createTopic(topicName, properties); + Optional topic = session.getTopic(topicName); + Assert.assertTrue(topic.isPresent()); + Assert.assertEquals(topicName, topic.get().getTopicName()); + // verify topic parameters + Assert.assertTrue(topic.get().getTopicAttributes().contains("path=root.**")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("start-time=2023-01-01")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("end-time=2023-12-31")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("format=TsFileHandler")); + + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testBasicCreateTopicIfNotExists() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + // create topic if not exits + String topicName = "topic3"; + session.createTopicIfNotExists(topicName); + Optional topic = session.getTopic(topicName); + Assert.assertTrue(topic.isPresent()); + Assert.assertEquals(topicName, topic.get().getTopicName()); + + // create topic if not exits + session.createTopicIfNotExists(topicName); + topic = session.getTopic(topicName); + Assert.assertTrue(topic.isPresent()); + Assert.assertEquals(topicName, topic.get().getTopicName()); + + // create topic if not exits + topicName = "topic4"; + Properties properties = new Properties(); + properties.put("path", "root.**"); + properties.put("start-time", "2023-01-01"); + properties.put("end-time", "2023-12-31"); + properties.put("format", "TsFileHandler"); + session.createTopicIfNotExists(topicName, properties); + topic = session.getTopic(topicName); + Assert.assertTrue(topic.isPresent()); + Assert.assertEquals(topicName, topic.get().getTopicName()); + // verify topic parameters + Assert.assertTrue(topic.get().getTopicAttributes().contains("path=root.**")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("start-time=2023-01-01")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("end-time=2023-12-31")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("format=TsFileHandler")); + + // create topic if not exits + properties.put("start-time", "2023-01-02"); + session.createTopicIfNotExists(topicName, properties); + topic = session.getTopic(topicName); + Assert.assertTrue(topic.isPresent()); + Assert.assertEquals(topicName, topic.get().getTopicName()); + // verify Topic Parameters + Assert.assertTrue(topic.get().getTopicAttributes().contains("path=root.**")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("start-time=2023-01-01")); + Assert.assertFalse(topic.get().getTopicAttributes().contains("start-time=2023-01-02")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("end-time=2023-12-31")); + Assert.assertTrue(topic.get().getTopicAttributes().contains("format=TsFileHandler")); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testBasicDropTopic() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + // create topic + String topicName = "topic5"; + session.createTopic(topicName); + + // drop topic + session.dropTopic(topicName); + Assert.assertFalse(session.getTopic(topicName).isPresent()); + + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testBasicDropTopicIfExists() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + // create topic + String topicName = "topic6"; + session.createTopic(topicName); + + // drop topic if exists + session.dropTopicIfExists(topicName); + Assert.assertFalse(session.getTopic(topicName).isPresent()); + + // drop topic if exists + session.dropTopicIfExists(topicName); + Assert.assertFalse(session.getTopic(topicName).isPresent()); + + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionIsolationIT.java new file mode 100644 index 0000000000000..7a53bf08744d8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionIsolationIT.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.local.tablemodel; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.ISubscriptionTreeSession; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumerBuilder; +import org.apache.iotdb.subscription.it.local.AbstractSubscriptionLocalIT; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBSubscriptionIsolationIT extends AbstractSubscriptionLocalIT { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void testTopicIsolation() throws Exception { + final String treeTopicName = "treeTopic"; + final String tableTopicName = "tableTopic"; + + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + // create tree topic + try (final ISubscriptionTreeSession session = + new SubscriptionTreeSessionBuilder().host(host).port(port).build()) { + session.open(); + session.createTopic(treeTopicName); + } + + // create table topic + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + session.createTopic(tableTopicName); + } + + // show topic on tree session + try (final ISubscriptionTreeSession session = + new SubscriptionTreeSessionBuilder().host(host).port(port).build()) { + session.open(); + Assert.assertEquals(1, session.getTopics().size()); + Assert.assertTrue(session.getTopic(treeTopicName).isPresent()); + Assert.assertFalse(session.getTopic(tableTopicName).isPresent()); + } + + // show topic on table session + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + Assert.assertEquals(1, session.getTopics().size()); + Assert.assertTrue(session.getTopic(tableTopicName).isPresent()); + Assert.assertFalse(session.getTopic(treeTopicName).isPresent()); + } + + // drop table topic on tree session + try (final ISubscriptionTreeSession session = + new SubscriptionTreeSessionBuilder().host(host).port(port).build()) { + session.open(); + try { + session.dropTopic(tableTopicName); + fail(); + } catch (final Exception ignored) { + } + } + + // drop tree topic on table session + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + try { + session.dropTopic(treeTopicName); + fail(); + } catch (final Exception ignored) { + } + } + + // drop tree topic on tree session + try (final ISubscriptionTreeSession session = + new SubscriptionTreeSessionBuilder().host(host).port(port).build()) { + session.open(); + session.dropTopic(treeTopicName); + } + + // drop table topic on table session + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + session.dropTopic(tableTopicName); + } + } + + @Test + public void testSubscriptionIsolation() throws Exception { + final String treeTopicName = "treeTopic"; + final String tableTopicName = "tableTopic"; + + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + // create tree topic + try (final ISubscriptionTreeSession session = + new SubscriptionTreeSessionBuilder().host(host).port(port).build()) { + session.open(); + session.createTopic(treeTopicName); + } + + // create table topic + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + session.createTopic(tableTopicName); + } + + // subscribe table topic on tree consumer + try (final ISubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumerBuilder().host(host).port(port).build()) { + consumer.open(); + try { + consumer.subscribe(tableTopicName); + fail(); + } catch (final Exception ignored) { + } + } + + // subscribe tree topic on table consumer + try (final ISubscriptionTablePullConsumer consumer = + new SubscriptionTablePullConsumerBuilder().host(host).port(port).build()) { + consumer.open(); + try { + consumer.subscribe(treeTopicName); + fail(); + } catch (final Exception ignored) { + } + } + + // subscribe tree topic on tree consumer + final ISubscriptionTreePullConsumer treeConsumer = + new SubscriptionTreePullConsumerBuilder().host(host).port(port).build(); + treeConsumer.open(); + treeConsumer.subscribe(treeTopicName); + + // subscribe table topic on table consumer + final ISubscriptionTablePullConsumer tableConsumer = + new SubscriptionTablePullConsumerBuilder().host(host).port(port).build(); + tableConsumer.open(); + tableConsumer.subscribe(tableTopicName); + + // show subscription on tree session + try (final ISubscriptionTreeSession session = + new SubscriptionTreeSessionBuilder().host(host).port(port).build()) { + session.open(); + Assert.assertEquals(1, session.getSubscriptions().size()); + Assert.assertEquals(1, session.getSubscriptions(treeTopicName).size()); + Assert.assertEquals(0, session.getSubscriptions(tableTopicName).size()); + } + + // show subscription on table session + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + Assert.assertEquals(1, session.getSubscriptions().size()); + Assert.assertEquals(1, session.getSubscriptions(tableTopicName).size()); + Assert.assertEquals(0, session.getSubscriptions(treeTopicName).size()); + } + + // unsubscribe table topic on tree consumer + try { + treeConsumer.unsubscribe(tableTopicName); + fail(); + } catch (final Exception ignored) { + + } + + // unsubscribe tree topic on table consumer + try { + tableConsumer.unsubscribe(treeTopicName); + fail(); + } catch (final Exception ignored) { + + } + + // close consumers + treeConsumer.close(); + tableConsumer.close(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java new file mode 100644 index 0000000000000..14bf3583ad13f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.local.tablemodel; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePushConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePushConsumerBuilder; +import org.apache.iotdb.session.subscription.model.Subscription; +import org.apache.iotdb.session.subscription.model.Topic; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.local.AbstractSubscriptionLocalIT; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBSubscriptionPermissionIT extends AbstractSubscriptionLocalIT { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void testMetaAccessControl() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + final String username = "thulab"; + final String password = "passwd"; + + // create user + createUser(EnvFactory.getEnv(), username, password); + + // root user + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + // create topic + final String topicName = "topic_root"; + session.createTopic(topicName); + Assert.assertTrue(session.getTopic(topicName).isPresent()); + Assert.assertEquals(topicName, session.getTopic(topicName).get().getTopicName()); + // show topic + final Optional topic = session.getTopic(topicName); + Assert.assertTrue(topic.isPresent()); + Assert.assertEquals(topicName, topic.get().getTopicName()); + // drop topic + session.dropTopic(topicName); + // show subscription + final Set subscriptions = session.getSubscriptions(topicName); + Assert.assertTrue(subscriptions.isEmpty()); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // normal user + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .build()) { + // create topic + final String topicName = "topic_thulab"; + session.createTopic(topicName); + fail(); + } catch (final Exception ignored) { + + } + + // normal user + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .build()) { + // show topics + session.getTopics(); + fail(); + } catch (final Exception ignored) { + + } + + // normal user + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .build()) { + // show subscriptions + session.getSubscriptions(); + fail(); + } catch (final Exception ignored) { + + } + } + + /** + * Tests runtime access control in the same consumer group. + * + *

In IoTDB subscriptions, all consumers in one group must use identical credentials when + * subscribing to the same topic. This test creates a topic and three consumers: + * + *

+ * + *

    + *
  • consumer1 and consumer2 use "thulab:passwd". + *
  • consumer3 uses "hacker:qwerty123". + *
+ * + *

Since consumer3 uses different credentials, it should be rejected. + */ + @Test + public void testRuntimeAccessControl() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + final String topicName = "topic1"; + + // create user + if (!TestUtils.tryExecuteNonQueriesWithRetry( + EnvFactory.getEnv(), + Arrays.asList("create user `thulab` 'passwd'", "create user `hacker` 'qwerty123'"))) { + return; + } + + // root user + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + // create topic + session.createTopic(topicName); + Assert.assertTrue(session.getTopic(topicName).isPresent()); + Assert.assertEquals(topicName, session.getTopic(topicName).get().getTopicName()); + } catch (final Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final AtomicInteger rowCount = new AtomicInteger(); + try (final ISubscriptionTablePushConsumer consumer1 = + new SubscriptionTablePushConsumerBuilder() + .host(host) + .port(port) + .username("thulab") + .password("passwd") + .consumerId("thulab_consumer_1") + .consumerGroupId("thulab_consumer_group") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .build(); + final ISubscriptionTablePushConsumer consumer2 = + new SubscriptionTablePushConsumerBuilder() + .host(host) + .port(port) + .username("thulab") + .password("passwd") + .consumerId("thulab_consumer_2") + .consumerGroupId("thulab_consumer_group") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .build(); + final ISubscriptionTablePushConsumer consumer3 = + new SubscriptionTablePushConsumerBuilder() + .host(host) + .port(port) + .username("hacker") + .password("qwerty123") + .consumerId("hacker_consumer") + .consumerGroupId("thulab_consumer_group") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .build()) { + + consumer1.open(); + consumer1.subscribe(topicName); + consumer2.open(); + consumer2.subscribe(topicName); + consumer3.open(); + consumer3.subscribe(topicName); + + fail(); + } catch (final Exception ignored) { + } + } + + /** + * Tests strict runtime access control in the same consumer group. + * + *

In IoTDB subscriptions, all consumers in one group must use identical credentials. This test + * creates two consumers with "thulab:passwd" and one with "hacker:qwerty123". Since the latter + * does not match the required credentials, it should be rejected. + */ + @Test + public void testStrictRuntimeAccessControl() { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + // create user + if (!TestUtils.tryExecuteNonQueriesWithRetry( + EnvFactory.getEnv(), + Arrays.asList("create user `thulab` 'passwd'", "create user `hacker` 'qwerty123'"))) { + return; + } + + final AtomicInteger rowCount = new AtomicInteger(); + try (final ISubscriptionTablePushConsumer consumer1 = + new SubscriptionTablePushConsumerBuilder() + .host(host) + .port(port) + .username("thulab") + .password("passwd") + .consumerId("thulab_consumer_1") + .consumerGroupId("thulab_consumer_group") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .build(); + final ISubscriptionTablePushConsumer consumer2 = + new SubscriptionTablePushConsumerBuilder() + .host(host) + .port(port) + .username("thulab") + .password("passwd") + .consumerId("thulab_consumer_2") + .consumerGroupId("thulab_consumer_group") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .build(); + final ISubscriptionTablePushConsumer consumer3 = + new SubscriptionTablePushConsumerBuilder() + .host(host) + .port(port) + .username("hacker") + .password("qwerty123") + .consumerId("hacker_consumer") + .consumerGroupId("thulab_consumer_group") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + dataSet.next(); + rowCount.addAndGet(1); + } + } + return ConsumeResult.SUCCESS; + }) + .build()) { + + consumer1.open(); + consumer2.open(); + consumer3.open(); + + fail(); + } catch (final Exception ignored) { + } + } + + @Test + public void testTablePermission() { + createUser(EnvFactory.getEnv(), "test", "test123"); + + assertTableNonQueryTestFail( + EnvFactory.getEnv(), + "create topic topic1", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableTestFail( + EnvFactory.getEnv(), + "show topics", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableTestFail( + EnvFactory.getEnv(), + "show subscriptions", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + assertTableNonQueryTestFail( + EnvFactory.getEnv(), + "drop topic topic1", + "803: Access Denied: No permissions for this operation, only root user is allowed", + "test", + "test123", + null); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java index d0d23a0407f0e..20b884bf70e37 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java @@ -21,13 +21,13 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.session.subscription.consumer.SubscriptionExecutorServiceManager; +import org.apache.iotdb.session.subscription.consumer.base.SubscriptionExecutorServiceManager; import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; import org.junit.After; import org.junit.Before; -abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionIT { +public abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionIT { protected BaseEnv sender; protected BaseEnv receiver1; @@ -35,7 +35,7 @@ abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionIT { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); // increase the number of threads to speed up testing @@ -61,18 +61,18 @@ protected void setUpConfig() { receiver2.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); // 10 min, assert that the operations will not time out - sender.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiver1.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); - receiver2.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + sender.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiver1.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiver2.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); } @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { sender.cleanClusterEnvironment(); receiver1.cleanClusterEnvironment(); receiver2.cleanClusterEnvironment(); + + super.tearDown(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/tablemodel/.gitkeep b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/tablemodel/.gitkeep new file mode 100644 index 0000000000000..585be9602fc3c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/tablemodel/.gitkeep @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/IoTDBSubscriptionSharingIT.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java rename to integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/IoTDBSubscriptionSharingIT.java index 0dc6edf1548c6..f92f8b0b2c3bc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/IoTDBSubscriptionSharingIT.java @@ -17,21 +17,23 @@ * under the License. */ -package org.apache.iotdb.subscription.it.triple; +package org.apache.iotdb.subscription.it.triple.treemodel; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeArchVerification; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.session.subscription.SubscriptionSession; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; import org.apache.iotdb.session.subscription.consumer.ConsumeResult; -import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.AbstractSubscriptionTripleIT; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi.IoTDBMultiGroupVsMultiConsumerIT; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TsFileReader; @@ -39,6 +41,7 @@ import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -59,8 +62,9 @@ import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; +/** refer to {@link IoTDBMultiGroupVsMultiConsumerIT} */ @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionTreeArchVerification.class}) public class IoTDBSubscriptionSharingIT extends AbstractSubscriptionTripleIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionSharingIT.class); @@ -79,8 +83,8 @@ public class IoTDBSubscriptionSharingIT extends AbstractSubscriptionTripleIT { private final String sql3 = "select count(s_0) from " + databasePrefix + "3.d_0"; private final String sql4 = "select count(s_0) from " + databasePrefix + "4.d_0"; - private final List schemaList = new ArrayList<>(2); - private final List consumers = new ArrayList<>(10); + private final List schemaList = new ArrayList<>(2); + private final List consumers = new ArrayList<>(10); private void createTopic( final String topicName, @@ -90,7 +94,7 @@ private void createTopic( final boolean isTsFile) { final String host = sender.getIP(); final int port = Integer.parseInt(sender.getPort()); - try (final SubscriptionSession session = new SubscriptionSession(host, port)) { + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { session.open(); final Properties properties = new Properties(); if (path != null) { @@ -122,10 +126,10 @@ private void insertData(long timestamp, final String device, final int rows) { final Tablet tablet = new Tablet(device, schemaList, rows); int rowIndex; for (int row = 0; row < rows; row++) { - rowIndex = tablet.rowSize++; + rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (row + 1) * 1400 + row); - tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (row + 1) * 100 + 0.5); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (row + 1) * 1400 + row); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (row + 1) * 100 + 0.5); timestamp += 2000; } session.insertTablet(tablet); @@ -186,7 +190,7 @@ private void insertDatum() { private void preparePushConsumers() { // create consumers consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_0") @@ -213,7 +217,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_1") @@ -240,7 +244,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_2") @@ -259,7 +263,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_3") @@ -278,7 +282,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_4") @@ -297,7 +301,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_5") @@ -316,7 +320,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_6") @@ -334,9 +338,8 @@ private void preparePushConsumers() { return ConsumeResult.SUCCESS; }) .buildPushConsumer()); - consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_7") @@ -378,7 +381,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_8") @@ -390,7 +393,7 @@ private void preparePushConsumers() { reader.query( QueryExpression.create( Collections.singletonList( - new Path(databasePrefix + 6 + ".d_0", "s_0", true)), + new Path(databasePrefix + "6.d_0", "s_0", true)), null)); while (dataset.hasNext()) { rowCount6.addAndGet(1); @@ -403,7 +406,7 @@ private void preparePushConsumers() { }) .buildPushConsumer()); consumers.add( - new SubscriptionPushConsumer.Builder() + new SubscriptionTreePushConsumer.Builder() .host(sender.getIP()) .port(Integer.parseInt(sender.getPort())) .consumerId("consumer_id_9") @@ -446,7 +449,7 @@ private void preparePushConsumers() { .buildPushConsumer()); // open consumers - for (final SubscriptionPushConsumer consumer : consumers) { + for (final SubscriptionTreePushConsumer consumer : consumers) { consumer.open(); } @@ -465,7 +468,7 @@ private void preparePushConsumers() { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); // prepare schemaList @@ -475,7 +478,7 @@ public void setUp() { @Override @After - public void tearDown() { + public void tearDown() throws Exception { // log some info try { LOGGER.info("[src] {} = {}", sql1, getCount(sender, sql1)); @@ -503,7 +506,7 @@ public void tearDown() { LOGGER.info("rowCount6 = {}", rowCount6.get()); // close consumers - for (final SubscriptionPushConsumer consumer : consumers) { + for (final SubscriptionTreePushConsumer consumer : consumers) { try { consumer.close(); } catch (final Exception ignored) { @@ -531,7 +534,8 @@ public void testSubscriptionSharing() { getCount(sender, sql1), getCount(receiver1, sql1) + getCount(receiver2, sql1)); // "c4,c6|topic2" - Assert.assertEquals(105, getCount(receiver1, sql2) + getCount(receiver2, sql2)); + Assert.assertEquals( + getCount(sender, sql2) - 400, getCount(receiver1, sql2) + getCount(receiver2, sql2)); // "c4,c5|c7,c9|topic3" final long topic3Total = getCount(receiver1, sql3) + getCount(receiver2, sql3); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/AbstractSubscriptionTreeRegressionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/AbstractSubscriptionTreeRegressionIT.java new file mode 100644 index 0000000000000..4b0164801cf18 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/AbstractSubscriptionTreeRegressionIT.java @@ -0,0 +1,475 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.subscription.it.triple.AbstractSubscriptionTripleIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS; + +public abstract class AbstractSubscriptionTreeRegressionIT extends AbstractSubscriptionTripleIT { + + private static final Logger LOGGER = + LoggerFactory.getLogger(AbstractSubscriptionTreeRegressionIT.class); + private static final String DROP_DATABASE_SQL = "drop database "; + + protected static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + + public String SRC_HOST; + public String DEST_HOST; + public String DEST_HOST2; + + public int SRC_PORT; + public int DEST_PORT; + public int DEST_PORT2; + + public SubscriptionTreeSession subs; + + public Session session_src; + public Session session_dest; + public Session session_dest2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + beforeSuite(); + } + + @Override + @After + public void tearDown() throws Exception { + afterSuite(); + super.tearDown(); + } + + public void beforeSuite() throws IoTDBConnectionException { + SRC_HOST = sender.getIP(); + DEST_HOST = receiver1.getIP(); + DEST_HOST2 = receiver2.getIP(); + + SRC_PORT = Integer.parseInt(sender.getPort()); + DEST_PORT = Integer.parseInt(receiver1.getPort()); + DEST_PORT2 = Integer.parseInt(receiver2.getPort()); + + session_src = + new Session.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .username("root") + .password("root") + .zoneId(ZoneId.of("Asia/Shanghai")) + .build(); + session_dest = + new Session.Builder() + .host(DEST_HOST) + .port(DEST_PORT) + .username("root") + .password("root") + .zoneId(ZoneId.of("Asia/Shanghai")) + .build(); + session_dest2 = + new Session.Builder() + .host(DEST_HOST2) + .port(DEST_PORT2) + .username("root") + .password("root") + .zoneId(ZoneId.of("Asia/Shanghai")) + .build(); + session_src.open(false); + session_dest.open(false); + session_dest2.open(false); + + subs = new SubscriptionTreeSession(SRC_HOST, SRC_PORT); + subs.open(); + System.out.println("TestConfig beforeClass"); + } + + public void afterSuite() throws IoTDBConnectionException { + System.out.println("TestConfig afterClass"); + session_src.close(); + session_dest.close(); + session_dest2.close(); + subs.close(); + } + + public void createDB(String database) throws IoTDBConnectionException { + try { + session_src.createDatabase(database); + } catch (StatementExecutionException e) { + } + try { + session_dest.createDatabase(database); + } catch (StatementExecutionException e) { + } + try { + session_dest2.createDatabase(database); + } catch (StatementExecutionException e) { + } + } + + public void dropDB(String pattern) throws IoTDBConnectionException { + try { + session_src.executeNonQueryStatement(DROP_DATABASE_SQL + pattern + "*"); + } catch (StatementExecutionException e) { + System.out.println("### src:" + e); + } + try { + session_dest.executeNonQueryStatement(DROP_DATABASE_SQL + pattern + "*"); + } catch (StatementExecutionException e) { + System.out.println("### dest:" + e); + } + try { + session_dest2.executeNonQueryStatement(DROP_DATABASE_SQL + pattern + "*"); + } catch (StatementExecutionException e) { + System.out.println("### dest2:" + e); + } + } + + public SubscriptionTreePullConsumer create_pull_consumer( + String groupId, String consumerId, Boolean autoCommit, Long interval) + throws TException, IoTDBConnectionException, IOException, StatementExecutionException { + SubscriptionTreePullConsumer pullConsumer; + Properties properties = new Properties(); + properties.put(ConsumerConstant.HOST_KEY, SRC_HOST); + properties.put(ConsumerConstant.PORT_KEY, SRC_PORT); + if (groupId != null) { + properties.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, groupId); + } + if (consumerId != null) { + properties.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); + } + if (autoCommit != null) { + properties.put(ConsumerConstant.AUTO_COMMIT_KEY, autoCommit); + } + if (interval != null) { + properties.put(ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_KEY, interval); + } + properties.put(ConsumerConstant.FILE_SAVE_DIR_KEY, "target/pull-subscription"); + pullConsumer = new SubscriptionTreePullConsumer(properties); + pullConsumer.open(); + return pullConsumer; + } + + public void createTopic_s( + String topicName, String pattern, String start, String end, boolean isTsfile) + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + if (pattern != null) { + properties.setProperty("path", pattern); + } + if (start != null) { + properties.setProperty("start-time", start); + } + if (end != null) { + properties.setProperty("end-time", end); + } + if (isTsfile) { + properties.setProperty("format", "TsFileHandler"); + } else { + properties.setProperty("format", "SessionDataSet"); + } + properties.setProperty("processor", "do-nothing-processor"); + subs.createTopic(topicName, properties); + } + + public void createTopic_s( + String topicName, + String pattern, + String start, + String end, + boolean isTsfile, + String mode, + String loose_range) + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + if (pattern != null) { + properties.setProperty(TopicConstant.PATH_KEY, pattern); + } + if (start != null) { + properties.setProperty(TopicConstant.START_TIME_KEY, start); + } + if (end != null) { + properties.setProperty(TopicConstant.END_TIME_KEY, end); + } + if (isTsfile) { + properties.setProperty(TopicConstant.FORMAT_KEY, "TsFileHandler"); + } else { + properties.setProperty(TopicConstant.FORMAT_KEY, "SessionDataSet"); + } + if (mode != null && !mode.isEmpty()) { + properties.setProperty(TopicConstant.MODE_KEY, mode); + } + if (loose_range != null && !loose_range.isEmpty()) { + properties.setProperty(TopicConstant.LOOSE_RANGE_KEY, loose_range); + } + subs.createTopic(topicName, properties); + } + + public static long getCount(Session session, String sql) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = session.executeQueryStatement(sql); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long result = rowRecord.getFields().get(0).getLongV(); + return result; + } + return 0; + } + + public void check_count(int expect_count, String sql, String msg) + throws IoTDBConnectionException, StatementExecutionException { + assertEquals(getCount(session_dest, sql), expect_count, "Query count:" + msg); + } + + public void check_count2(int expect_count, String sql, String msg) + throws IoTDBConnectionException, StatementExecutionException { + assertEquals(getCount(session_dest2, sql), expect_count, "Query count:" + msg); + } + + public void consume_data(SubscriptionTreePullConsumer consumer, Session session) + throws TException, + IOException, + StatementExecutionException, + InterruptedException, + IoTDBConnectionException { + while (true) { + Thread.sleep(1000); + + List messages = consumer.poll(Duration.ofMillis(POLL_TIMEOUT_MS)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + } + } + consumer.commitSync(messages); + } + } + + public List consume_tsfile_withFileCount( + SubscriptionTreePullConsumer consumer, String device) throws InterruptedException { + return consume_tsfile(consumer, Collections.singletonList(device)); + } + + public int consume_tsfile(SubscriptionTreePullConsumer consumer, String device) + throws InterruptedException { + return consume_tsfile(consumer, Collections.singletonList(device)).get(0); + } + + public List consume_tsfile(SubscriptionTreePullConsumer consumer, List devices) + throws InterruptedException { + List rowCounts = new ArrayList<>(devices.size()); + for (int i = 0; i < devices.size(); i++) { + rowCounts.add(new AtomicInteger(0)); + } + AtomicInteger onReceived = new AtomicInteger(0); + while (true) { + Thread.sleep(1000); + // That is, the consumer poll will keep pulling if no messages are fetched within the timeout, + // until a message is fetched or the time exceeds the timeout. + List messages = consumer.poll(Duration.ofMillis(POLL_TIMEOUT_MS)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + onReceived.incrementAndGet(); + // System.out.println(FORMAT.format(new Date()) + " onReceived=" + onReceived.get()); + final SubscriptionTsFileHandler tsFileHandler = message.getTsFileHandler(); + try (final TsFileReader tsFileReader = tsFileHandler.openReader()) { + for (int i = 0; i < devices.size(); i++) { + final Path path = new Path(devices.get(i), "s_0", true); + final QueryDataSet dataSet = + tsFileReader.query(QueryExpression.create(Collections.singletonList(path), null)); + while (dataSet.hasNext()) { + RowRecord next = dataSet.next(); + rowCounts.get(i).addAndGet(1); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + // System.out.println(FORMAT.format(new Date()) + " consume tsfile " + i + ":" + + // rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + consumer.commitSync(messages); + } + List results = new ArrayList<>(devices.size()); + for (AtomicInteger rowCount : rowCounts) { + results.add(rowCount.get()); + } + results.add(onReceived.get()); + return results; + } + + public static void consume_data_long( + SubscriptionTreePullConsumer consumer, Session session, Long timeout) + throws StatementExecutionException, InterruptedException, IoTDBConnectionException { + timeout = System.currentTimeMillis() + timeout; + while (System.currentTimeMillis() < timeout) { + List messages = consumer.poll(Duration.ofMillis(POLL_TIMEOUT_MS)); + if (messages.isEmpty()) { + Thread.sleep(1000); + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + } + } + consumer.commitSync(messages); + } + } + + public void consume_data(SubscriptionTreePullConsumer consumer) + throws TException, + IOException, + StatementExecutionException, + InterruptedException, + IoTDBConnectionException { + consume_data(consumer, session_dest); + } + + //////////////////////////// strict assertions //////////////////////////// + + public static void assertEquals(int actual, int expected) { + Assert.assertEquals(expected, actual); + } + + public static void assertEquals(Integer actual, int expected) { + Assert.assertEquals(expected, (Object) actual); + } + + public static void assertEquals(int actual, Integer expected) { + Assert.assertEquals((Object) expected, actual); + } + + public static void assertEquals(Integer actual, Integer expected) { + Assert.assertEquals(expected, actual); + } + + public static void assertEquals(int actual, int expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertEquals(Integer actual, int expected, String message) { + Assert.assertEquals(message, expected, (Object) actual); + } + + public static void assertEquals(int actual, Integer expected, String message) { + Assert.assertEquals(message, (Object) expected, actual); + } + + public static void assertEquals(Integer actual, Integer expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertEquals(long actual, long expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertEquals(boolean actual, boolean expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertTrue(boolean condition) { + Assert.assertTrue(condition); + } + + public static void assertTrue(boolean condition, String message) { + Assert.assertTrue(message, condition); + } + + public static void assertFalse(boolean condition) { + Assert.assertFalse(condition); + } + + public static void assertFalse(boolean condition, String message) { + Assert.assertFalse(message, condition); + } + + //////////////////////////// non-strict assertions //////////////////////////// + + public static void assertGte(int actual, int expected) { + assertGte(actual, expected, null); + } + + public static void assertGte(int actual, int expected, String message) { + assertGte((long) actual, expected, message); + } + + public static void assertGte(long actual, long expected, String message) { + assertTrue(actual >= expected, message); + if (!(actual == expected)) { + String skipMessage = actual + " should be equals to " + expected; + if (Objects.nonNull(message)) { + skipMessage += ", message: " + message; + } + LOGGER.warn(skipMessage); + Assume.assumeTrue(skipMessage, actual == expected); + } + } + + public void check_count_non_strict(int expect_count, String sql, String msg) + throws IoTDBConnectionException, StatementExecutionException { + assertGte(getCount(session_dest, sql), expect_count, "Query count: " + msg); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBDefaultPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBDefaultPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..d6681dee040b9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBDefaultPullConsumerDataSetIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBDefaultPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + public static SubscriptionTreePullConsumer consumer; + private int deviceCount = 3; + private static final String databasePrefix = "root.DefaultPullConsumerDataSet"; + private static String topicName = "topic_autodb_DefaultPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, null, null, null, false); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("create database " + databasePrefix + i); + session_dest.executeNonQueryStatement("create database " + databasePrefix + i); + } + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("drop database " + databasePrefix + i); + session_dest.executeNonQueryStatement("drop database " + databasePrefix + i); + } + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + List devices = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount; i++) { + devices.add(databasePrefix + i + ".d_0"); + } + consumer = create_pull_consumer("pull_auto_create_db", "default_pattern_dataset", false, null); + for (int i = 0; i < deviceCount; i++) { + // Write data before subscribing + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + // Subscribe + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + for (int i = 0; i < deviceCount; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + } + String sql = "select count(s_0) from " + databasePrefix + "0.d_0"; + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + // Consumption data + AWAIT.untilAsserted( + () -> { + session_src.executeNonQueryStatement("flush"); + consume_data(consumer, session_dest); + for (int i = 0; i < deviceCount; i++) { + check_count( + 10, "select count(s_0) from " + devices.get(i), i + ":Consumption Data:s_0"); + } + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Unsubscribe and then write data + for (int i = 0; i < deviceCount; i++) { + insert_data(1707782400000L, devices.get(i)); // 2024-02-13 08:00:00+08:00 + } + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + session_src.executeNonQueryStatement("flush"); + consume_data(consumer, session_dest); + for (int i = 0; i < deviceCount; i++) { + check_count( + 15, "select count(s_0) from " + devices.get(i), i + ":consume data again:s_0"); + } + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBDefaultTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBDefaultTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..f99d88b8a29c8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBDefaultTsfilePushConsumerIT.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: root.** + * TsFile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBDefaultTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private SubscriptionTreePushConsumer consumer; + private int deviceCount = 3; + private static final String databasePrefix = "root.DefaultTsfilePushConsumer"; + private static String topicName = "topicDefaultTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, null, null, null, true); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("create database " + databasePrefix + i); + } + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("drop database " + databasePrefix + i); + } + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + List devices = new ArrayList<>(deviceCount); + List paths = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount; i++) { + devices.add(databasePrefix + i + ".d_0"); + paths.add(new Path(devices.get(i), "s_0", true)); + } + System.out.println("### Before Subscription Write Data ###"); + for (int i = 0; i < deviceCount; i++) { + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + session_src.executeNonQueryStatement("flush"); + final AtomicInteger onReceiveCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount; i++) { + rowCounts.add(new AtomicInteger(0)); + } + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("default_pattern_TsFile_consumer") + .consumerGroupId("push_auto_create_db") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + System.out.println( + FORMAT.format(new Date()) + " ######## onReceived: " + onReceiveCount.get()); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < deviceCount; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + // System.out.println(format.format(new + // Date())+" "+next.getTimestamp()+","+next.getFields()); + } + System.out.println( + FORMAT.format(new Date()) + + " rowCounts_" + + i + + ":" + + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "After subscribing: show subscriptions"); + for (int i = 0; i < deviceCount; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + System.out.println( + FORMAT.format(new Date()) + + " src " + + i + + ":" + + getCount(session_src, "select count(s_0) from " + devices.get(i))); + } + session_src.executeNonQueryStatement("flush"); + AWAIT.untilAsserted( + () -> { + for (int i = 0; i < deviceCount; i++) { + assertEquals(rowCounts.get(i).get(), 10, devices.get(i) + ".s_0"); + } + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + System.out.println("### Subscribe and write data ###"); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "After subscribing again: show subscriptions"); + for (int i = 0; i < deviceCount; i++) { + insert_data(1707782400000L, devices.get(i)); // 2024-02-13 08:00:00+08:00 + System.out.println( + FORMAT.format(new Date()) + + " src " + + i + + ":" + + getCount(session_src, "select count(s_0) from " + devices.get(i))); + } + session_src.executeNonQueryStatement("flush"); + + // Unsubscribe, then it will consume all again. + AWAIT.untilAsserted( + () -> { + for (int i = 0; i < deviceCount; i++) { + assertEquals(rowCounts.get(i).get(), 25, devices.get(i) + ".s_0"); + } + }); + System.out.println(FORMAT.format(new Date()) + " onReceived: " + onReceiveCount.get()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBRootDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBRootDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..8adefca6ce497 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBRootDatasetPushConsumerIT.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: root + * DataSet + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBRootDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String pattern = "root.**"; + public static SubscriptionTreePushConsumer consumer; + private int deviceCount = 3; + private static final String databasePrefix = "root.RootDatasetPushConsumer"; + private static final String database2 = "root.RootDatasetPushConsumer2.test"; + private static String topicName = "topicAutoCreateDB_RootDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, false); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(databasePrefix + "*.*"); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + List devices = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount - 1; i++) { + devices.add(databasePrefix + i + ".d_0"); + } + devices.add(database2 + ".d_2"); + for (int i = 0; i < deviceCount; i++) { + // Write data before subscribing + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_auto_create_db") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions(topicName).size(), 1, "subscribe:show subscriptions"); + for (int i = 0; i < deviceCount; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + } + String sql = "select count(s_0) from " + databasePrefix + "0.d_0"; + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + AWAIT.untilAsserted( + () -> { + check_count(10, "select count(s_0) from " + devices.get(0), "0:consume data:s_0"); + for (int i = 1; i < deviceCount; i++) { + check_count(10, "select count(s_0) from " + devices.get(i), i + ":consume data:s_0"); + } + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + for (int i = 0; i < deviceCount; i++) { + insert_data(1707782400000L, devices.get(i)); // 2024-02-13 08:00:00+08:00 + } + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "After subscribing again: show subscriptions"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count( + 15, + "select count(s_0) from " + devices.get(0), + "0:After subscribing again:consume data:s_0"); + for (int i = 1; i < deviceCount; i++) { + check_count( + 15, + "select count(s_0) from " + devices.get(i), + i + ":After subscribing again:consume data:s_0"); + } + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBRootPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBRootPullConsumeTsfileIT.java new file mode 100644 index 0000000000000..d340363a41bec --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/auto_create_db/IoTDBRootPullConsumeTsfileIT.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: db + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBRootPullConsumeTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String pattern = "root.**"; + private static final String device = "root.auto_create_db.RootPullConsumeTsfile.d_0"; + private static final String device2 = "root.RootPullConsumeTsfile.d_1"; + public static SubscriptionTreePullConsumer consumer; + private static String topicName = "topicAutoCreateDB_RootPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, true); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + session_src.executeNonQueryStatement("create database root.auto_create_db"); + session_src.executeNonQueryStatement("create database root.RootPullConsumeTsfile"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + session_src.executeNonQueryStatement("drop database root.auto_create_db"); + session_src.executeNonQueryStatement("drop database root.RootPullConsumeTsfile"); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_tsfile") + .consumerGroupId("pull_auto_create_db") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals(subs.getSubscriptions(topicName).size(), 1, "subscribe:show subscriptions"); + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + List devices = new ArrayList<>(2); + devices.add(device); + devices.add(device2); + List results = consume_tsfile(consumer, devices); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 10); + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions(topicName).size(), 0, "unsubscribe:show subscriptions"); + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "subscribe again:show subscriptions"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + results = consume_tsfile(consumer, devices); + assertEquals( + results.get(0), + 15, + "Unsubscribing and then re-subscribing will not retain progress. Full synchronization."); + assertEquals( + results.get(1), + 15, + "Unsubscribing and then re-subscribing will not retain progress. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/mix/IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/mix/IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT.java new file mode 100644 index 0000000000000..0eb3bceec3c08 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/mix/IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.mix; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: db + * Dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT + extends AbstractSubscriptionTreeRegressionIT { + private static String topicName = "`1-group.1-consumer.db`"; + private static List schemaList = new ArrayList<>(); + private final String database = "root.PushConsumerPullConsumerWith1TopicShareProcessMix"; + private final String device = database + ".d_0"; + private final String pattern = database + ".**"; + private SubscriptionTreePushConsumer consumer; + private SubscriptionTreePullConsumer consumer2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, false); + createDB(database); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + consumer2.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 100; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("dataset_push_consumer_1") + .consumerGroupId("db_pull_push_mix") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + + consumer2 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("dataset_pull_consumer_2") + .consumerGroupId("db_pull_push_mix") + .buildPullConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); + + thread.start(); + thread.join(); + consume_data(consumer2, session_dest2); + System.out.println("After subscribing:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions"); + + Thread.sleep(3000); + // The first 5 entries may have duplicate data + String sql = "select count(s_0) from " + device; + System.out.println("src push consumer: " + getCount(session_src, sql)); + System.out.println("dest push consumer: " + getCount(session_dest, sql)); + System.out.println("dest2 pull consumer: " + getCount(session_dest2, sql)); + AWAIT.untilAsserted( + () -> { + assertEquals( + getCount(session_dest, sql) + getCount(session_dest2, sql), + getCount(session_src, sql), + "share process"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamPullConsumerIT.java new file mode 100644 index 0000000000000..628bc858884b8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamPullConsumerIT.java @@ -0,0 +1,545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.param; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionIdentifierSemanticException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBTestParamPullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static SubscriptionTreePullConsumer consumer; + private static final String topicName = "TestParamPullConsumerTopic1"; + private static final String database = "root.TestParamPullConsumer"; + private static final String device = database + ".d_0"; + private final String pattern = "root.**"; + private static List schemaList = new ArrayList<>(); + + static { + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("TestParamPullConsumer_1") + .buildPullConsumer(); + consumer.open(); + createDB(database); + createTopic_s(topicName, null, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + subs.getTopics().forEach(System.out::println); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + session_src.executeNonQueryStatement("create user user02 'user02';"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + try { + session_src.executeNonQueryStatement("drop user user02"); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + private void run_single(SubscriptionTreePullConsumer consumer, int index) + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "subscribe then show subscriptions:" + index); + Long timestamp = 1706659200000L + 8000 * index; + // Write data + insert_data(timestamp); + // consume + consume_data(consumer, session_dest); + // Unsubscribe + consumer.unsubscribe(topicName); + consumer.close(); + check_count( + 4, + "select count(s_0) from " + device + " where time >= " + timestamp, + "Consumption data:" + pattern); + } + + @Test + public void testUnsetGroup() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .autoCommit(true) + .autoCommitIntervalMs(1000L) + .buildPullConsumer(); + run_single(consumer1, 1); + } + + @Test + public void testUnsetConsumer() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerGroupId("TestParamPullConsumer_group_id_1") + .autoCommit(true) + .autoCommitIntervalMs(1000L) + .buildPullConsumer(); + run_single(consumer1, 2); + } + + @Test + public void testUnsetConsumerGroup() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .autoCommit(true) + .autoCommitIntervalMs(1000L) + .buildPullConsumer(); + run_single(consumer1, 3); + } + + @Test + public void testAutoCommitIntervalNegative() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("TestParamPullConsumer") + .autoCommitIntervalMs(-1) + .buildPullConsumer(); + run_single(consumer1, 5); + } + + @Test + public void testDuplicateConsumerId() { + SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testDuplicateConsumerId") + .consumerGroupId("TestParamPullConsumer_1") + .autoCommitIntervalMs(-1) + .buildPullConsumer(); + SubscriptionTreePullConsumer consumer2 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testDuplicateConsumerId") + .consumerGroupId("TestParamPullConsumer_2") + .autoCommitIntervalMs(-1) + .buildPullConsumer(); + SubscriptionTreePullConsumer consumer3 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testDuplicateConsumerId") + .consumerGroupId("TestParamPullConsumer_1") + .autoCommitIntervalMs(-1) + .buildPullConsumer(); + consumer.open(); + consumer2.open(); + consumer3.open(); + consumer.close(); + consumer2.close(); + consumer3.close(); + } + + @Test + public void testNodeUrls() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .nodeUrls(Collections.singletonList(SRC_HOST + ":" + SRC_PORT)) + .consumerId("testNodeUrls") + .consumerGroupId("TestParamPullConsumer") + .autoCommitIntervalMs(500L) + .buildPullConsumer(); + run_single(consumer1, 5); + } + + @Test(expected = NullPointerException.class) + public void testCreateConsumer_null() { + new SubscriptionTreePullConsumer(null).open(); + } + + @Test( + expected = + SubscriptionConnectionException.class) // connect to TEndPoint(ip:localhost, port:6667) + public void testCreateConsumer_empty() { + new SubscriptionTreePullConsumer(new Properties()).open(); + } + + @Test(expected = SubscriptionConnectionException.class) + public void testCreateConsumer_empty2() { + new SubscriptionTreePullConsumer.Builder().buildPullConsumer().open(); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testSubscribe_null() { + consumer.subscribe((String) null); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testSubscribe_empty() { + consumer.subscribe(""); + } + + @Test(expected = SubscriptionRuntimeCriticalException.class) + public void testSubscribe_notTopic() { + consumer.subscribe("topic_notCreate"); + } + + @Test + public void testSubscribe_dup() { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testSubscribe_dup") + .consumerGroupId("TestParamPullConsumer") + .buildPullConsumer(); + consumer1.open(); + consumer1.subscribe(topicName); + consumer1.subscribe(topicName); + consumer1.close(); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testUnSubscribe_null() { + consumer.unsubscribe((String) null); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testUnSubscribe_empty() { + consumer.unsubscribe(""); + } + + @Test(expected = SubscriptionRuntimeCriticalException.class) + public void testUnSubscribe_notTopic() { + consumer.unsubscribe("topic_notCreate"); + } + + @Test + public void testUnSubscribe_dup() { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testUnSubscribe_dup") + .consumerGroupId("TestParamPullConsumer") + .buildPullConsumer(); + consumer1.open(); + consumer1.subscribe(topicName); + consumer1.unsubscribe(topicName); + consumer1.unsubscribe(topicName); + consumer1.close(); + } + + @Test + public void testUnSubscribe_notSubs() + throws StatementExecutionException, IoTDBConnectionException { + subs.createTopic("t"); + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testUnSubscribe_notSubs") + .consumerGroupId("TestParamPullConsumer") + .buildPullConsumer(); + consumer1.open(); + // No subscription, unsubscribe directly + consumer1.unsubscribe("t"); + consumer1.close(); + subs.dropTopic("t"); + } + + @Test(expected = SubscriptionException.class) + public void testSubscribe_AfterClose() { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testSubscribe_AfterClose") + .consumerGroupId("TestParamPullConsumer") + .buildPullConsumer(); + consumer1.open(); + consumer1.subscribe(topicName); + consumer1.close(); + consumer1.subscribe(topicName); + } + + @Test(expected = SubscriptionException.class) + public void testUnSubscribe_AfterClose() { + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("testUnSubscribe_AfterClose") + .consumerGroupId("TestParamPullConsumer") + .buildPullConsumer(); + consumer1.open(); + consumer1.close(); + consumer1.unsubscribe(topicName); + } + + @Test(expected = SubscriptionConnectionException.class) + public void testNoUser() { + String userName = "user01"; + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .username(userName) + .buildPullConsumer() + .open(); + } + + @Test(expected = SubscriptionConnectionException.class) + public void testErrorPasswd() { + String userName = "user02"; + SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .username(userName) + .buildPullConsumer(); + consumer1.open(); + consumer1.close(); + } + + @Test + public void testTsfile_ts() + throws IoTDBConnectionException, + StatementExecutionException, + InterruptedException, + IOException { + String t1 = "tsTopic"; + createTopic_s(t1, database + ".d_0.s_0", null, null, true); + try (SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .buildPullConsumer()) { + consumer1.open(); + consumer1.subscribe(t1); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_0(time,s_0,s_1) values (1,10,20),(1000,30,60);"); + session_src.executeNonQueryStatement("flush"); + Thread.sleep(3000L); + AtomicInteger rowCount = new AtomicInteger(0); + while (true) { + Thread.sleep(1000L); + List messages = consumer1.poll(Duration.ofMillis(1000)); + if (messages.isEmpty()) { + break; + } + } + } + subs.dropTopic(t1); + } + + @Test + public void testTsfile_ts_normal() + throws IoTDBConnectionException, + StatementExecutionException, + InterruptedException, + IOException { + String t1 = "tsTopicNormal"; + String device = database + ".d_1"; + createTopic_s(t1, device + ".s_0", null, null, true); + try (SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .buildPullConsumer()) { + consumer1.open(); + consumer1.subscribe(t1); + session_src.executeNonQueryStatement( + "insert into " + device + "(time,s_0) values (1,10),(1000,30);"); + session_src.executeNonQueryStatement("flush"); + Thread.sleep(3000L); + AtomicInteger rowCount = new AtomicInteger(0); + while (true) { + Thread.sleep(1000L); + List messages = consumer1.poll(Duration.ofMillis(1000)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_0", true)), null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(device + ":" + next.getTimestamp() + "," + next.getFields()); + } + } + consumer1.commitSync(messages); + } + } + subs.dropTopic(t1); + } + + @Test + public void testTsfile_device() + throws IoTDBConnectionException, + StatementExecutionException, + InterruptedException, + IOException { + String t1 = "DeviceTopic"; + String device = database + ".d2"; + createTopic_s(t1, device + ".**", null, null, true); + try (SubscriptionTreePullConsumer consumer1 = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .buildPullConsumer()) { + consumer1.open(); + consumer1.subscribe(t1); + session_src.executeNonQueryStatement( + "insert into " + device + "(time,s_0,s_1) values (1,10,20),(1000,30,60);"); + session_src.executeNonQueryStatement("flush"); + Thread.sleep(3000L); + AtomicInteger rowCount = new AtomicInteger(0); + while (true) { + Thread.sleep(1000L); + List messages = consumer1.poll(Duration.ofMillis(1000)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_0", true)), null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(device + ":" + next.getTimestamp() + "," + next.getFields()); + } + } + consumer1.commitSync(messages); + } + } + subs.dropTopic(t1); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamPushConsumerIT.java new file mode 100644 index 0000000000000..8f138712fd404 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamPushConsumerIT.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.param; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionIdentifierSemanticException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBTestParamPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static SubscriptionTreePushConsumer consumer; + private static final String topicName = "TestParamPushConsumerTopic1"; + private static final String database = "root.TestParamPushConsumer"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + private String sql = "select count(s_0) from " + device; + + static { + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, null, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + subs.getTopics().forEach(System.out::println); + session_src.executeNonQueryStatement("create user user02 'user02';"); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_default") + .consumerGroupId("TestParamPushConsumer") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + System.out.println(message.getMessageType()); + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + try { + session_src.executeNonQueryStatement("drop user user02"); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void testUnsetGroupConsumer() + throws IoTDBConnectionException, StatementExecutionException { + long count = getCount(session_src, sql); + insert_data(1706659200000L); + try (final SubscriptionTreePushConsumer consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + // System.out.println("#### " + dataSet.getTablet().rowSize); + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + AWAIT.untilAsserted( + () -> { + check_count(5, sql, "before count=" + count); + }); + } + } + + @Test + public void testAlterPollTime() throws IoTDBConnectionException, StatementExecutionException { + String sql = "select count(s_0) from " + device; + long count = getCount(session_src, sql); + insert_data(1706669800000L); + try (final SubscriptionTreePushConsumer consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .autoPollTimeoutMs(1000) + .autoPollIntervalMs(10) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + AWAIT.untilAsserted( + () -> { + System.out.println(getCount(session_dest, sql)); + check_count(5, sql, " would sync all data including deleted, before count=" + count); + }); + } + } + + @Test(expected = NullPointerException.class) + public void testCreateConsumer_null() { + final Properties properties = null; + new SubscriptionTreePushConsumer(properties).open(); + } + + @Test( + expected = + SubscriptionConnectionException.class) // connect to TEndPoint(ip:localhost, port:6667) + public void testCreateConsumer_empty() { + new SubscriptionTreePushConsumer(new Properties()).open(); + } + + @Test(expected = SubscriptionConnectionException.class) + public void testCreateConsumer_empty2() { + new SubscriptionTreePushConsumer.Builder().buildPushConsumer().open(); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testSubscribe_null() { + consumer.subscribe((String) null); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testSubscribe_empty() { + consumer.subscribe(""); + } + + @Test(expected = SubscriptionRuntimeCriticalException.class) + public void testSubscribe_notTopic() { + consumer.subscribe("topic_notCreate"); + } + + @Test + public void testSubscribe_dup() { + SubscriptionTreePushConsumer consumer1 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_default") + .consumerGroupId("TestParamPushConsumer") + .buildPushConsumer(); + consumer1.open(); + consumer1.subscribe(topicName); + consumer1.subscribe(topicName); + consumer1.close(); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testUnSubscribe_null() { + consumer.unsubscribe((String) null); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testUnSubscribe_empty() { + consumer.unsubscribe(""); + } + + @Test(expected = SubscriptionRuntimeCriticalException.class) + public void testUnSubscribe_notTopic() { + consumer.unsubscribe("topic_notCreate"); + } + + @Test + public void testUnSubscribe_dup() { + SubscriptionTreePushConsumer consumer1 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("push_param_group_id_1") + .buildPushConsumer(); + consumer1.open(); + consumer1.subscribe(topicName); + consumer1.unsubscribe(topicName); + consumer1.unsubscribe(topicName); + consumer1.close(); + } + + @Test + public void testUnSubscribe_notSubs() + throws IoTDBConnectionException, StatementExecutionException { + subs.createTopic("t"); + SubscriptionTreePushConsumer consumer1 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("push_param_group_id_1") + .buildPushConsumer(); + consumer1.open(); + // No subscription, unsubscribe directly + consumer1.unsubscribe("t"); + consumer1.close(); + subs.dropTopic("t"); + } + + @Test(expected = SubscriptionException.class) + public void testSubscribe_AfterClose() { + SubscriptionTreePushConsumer consumer1 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("push_param_group_id_1") + .buildPushConsumer(); + consumer1.open(); + consumer1.subscribe(topicName); + consumer1.close(); + consumer1.subscribe(topicName); + } + + @Test(expected = SubscriptionException.class) + public void testUnSubscribe_AfterClose() { + SubscriptionTreePushConsumer consumer1 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("push_param_group_id_1") + .buildPushConsumer(); + consumer1.open(); + consumer1.close(); + consumer1.unsubscribe(topicName); + } + + @Test(expected = SubscriptionConnectionException.class) + public void testNoUser() { + String userName = "user01"; + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .username(userName) + .buildPushConsumer() + .open(); + } + + @Test(expected = SubscriptionConnectionException.class) + public void testErrorPasswd() { + String userName = "user02"; + SubscriptionTreePushConsumer consumer1 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .username(userName) + .buildPushConsumer(); + consumer1.open(); + consumer1.close(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamSubscriptionSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamSubscriptionSessionIT.java new file mode 100644 index 0000000000000..10783fc7736cf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamSubscriptionSessionIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.param; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBTestParamSubscriptionSessionIT extends AbstractSubscriptionTreeRegressionIT { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void testCreateSession_null_host() { + new SubscriptionTreeSession.Builder().host(null).build(); + } + + @Test(expected = IoTDBConnectionException.class) + public void testCreateSession_error_port() throws IoTDBConnectionException { + new SubscriptionTreeSession(SRC_HOST, SRC_PORT + 1).open(); + } + + @Test(expected = IoTDBConnectionException.class) + public void testCreateSession_ErrorHostname() throws IoTDBConnectionException { + new SubscriptionTreeSession.Builder().host("noName").build().open(); + } + + @Test(expected = IoTDBConnectionException.class) + public void testCreateSession_ErrorUsername() throws IoTDBConnectionException { + new SubscriptionTreeSession.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .username("admin") + .build() + .open(); + } + + @Test(expected = IoTDBConnectionException.class) + public void testCreateSession_ErrorPassword() throws IoTDBConnectionException { + new SubscriptionTreeSession.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .password("admin") + .build() + .open(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamTopicIT.java new file mode 100644 index 0000000000000..3906e8fea6af6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/param/IoTDBTestParamTopicIT.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.param; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionIdentifierSemanticException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Properties; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBTestParamTopicIT extends AbstractSubscriptionTreeRegressionIT { + private static SubscriptionTreePullConsumer consumer; + private static final String topicName = "TopicParam"; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerGroupId("g_TestParamTopic") + .consumerId("c1") + .buildPullConsumer(); + consumer.open(); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.getTopics() + .forEach( + topic -> { + try { + subs.dropTopic(topic.getTopicName()); + } catch (Exception ignored) { + } + }); + super.tearDown(); + } + + private void printTopics(String msg) + throws IoTDBConnectionException, StatementExecutionException { + System.out.println(msg); + subs.getTopics().forEach(System.out::println); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testCreateTopic_null() throws IoTDBConnectionException, StatementExecutionException { + subs.createTopic(null); + printTopics("testCreateTopic_null"); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testCreateTopic_emptyString() + throws IoTDBConnectionException, StatementExecutionException { + subs.createTopic(""); + printTopics("testCreateTopic_emptyString"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_dup() throws IoTDBConnectionException, StatementExecutionException { + subs.createTopic(topicName); + subs.createTopic(topicName); + printTopics("testCreateTopic_dup"); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testCreateTopic_invalid() + throws IoTDBConnectionException, StatementExecutionException { + subs.createTopic("Topic-1"); + printTopics("testCreateTopic_invalid"); + } + + @Test(expected = StatementExecutionException.class) // path filter conditions are not checked + public void testCreateTopic_invalidPath_no_root() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.PATH_KEY, "abc"); + subs.createTopic("topic_error_path1", properties); + printTopics("testCreateTopic_invalidPath_no_root"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_invalidPath_endWithPoint() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.PATH_KEY, "root."); + subs.createTopic("topic_error_path2", properties); + printTopics("testCreateTopic_invalidPath_endWithPoint"); + } + + @Test // Need to pay attention to the default value + public void testCreateTopic_invalidFormat() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.FORMAT_KEY, "abd"); + subs.createTopic("topic_error_path3", properties); + printTopics("testCreateTopic_invalidFormat"); + subs.dropTopic("topic_error_path3"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_invalidTime() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "abd"); + subs.createTopic("topic_error_path4", properties); + printTopics("testCreateTopic_invalidTime"); + } + + @Test + public void testCreateTopic_invalidTime2() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "now"); + properties.put(TopicConstant.END_TIME_KEY, "now"); + subs.createTopic("topic_error_path5", properties); + printTopics("testCreateTopic_invalidTime2"); + subs.dropTopic("topic_error_path5"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_invalidTime3() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "2024-01-01"); + properties.put(TopicConstant.END_TIME_KEY, "2023-01-01"); + subs.createTopic("topic_error_path6", properties); + printTopics("testCreateTopic_invalidTime3"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_invalidTime4() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "now"); + properties.put(TopicConstant.END_TIME_KEY, "2024-01-01"); + subs.createTopic("topic_error_path7", properties); + printTopics("testCreateTopic_invalidTime4"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_invalidTime5() + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "2023-01-32"); + properties.put(TopicConstant.END_TIME_KEY, "now"); + subs.createTopic("topic_error_path7", properties); + printTopics("testCreateTopic_invalidTime5"); + } + + @Test(expected = StatementExecutionException.class) + public void testCreateTopic_invalidTime6() + throws IoTDBConnectionException, + StatementExecutionException, + TException, + IOException, + InterruptedException { + Properties properties = new Properties(); + properties.put(TopicConstant.START_TIME_KEY, "2023-02-29"); + properties.put(TopicConstant.END_TIME_KEY, "now"); + subs.createTopic("topic_error_path8", properties); + printTopics("testCreateTopic_invalidTime6"); + consumer.subscribe("topic_error_path8"); + String database = "root.testClient"; + createDB(database); + session_src.executeNonQueryStatement("create timeseries " + database + ".d_1.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database + ".d_1.s_0 int32;"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time, s_0) values(1677628800000,33);"); + consume_data(consumer, session_dest); + check_count(1, "select count(s_0) from " + database + ".d_1;", "invalid date"); + consumer.unsubscribe("topic_error_path8"); + subs.dropTopic("topic_error_path8"); + dropDB(database); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testDropTopic_null() throws IoTDBConnectionException, StatementExecutionException { + subs.dropTopic(null); + } + + @Test(expected = SubscriptionIdentifierSemanticException.class) + public void testDropTopic_empty() throws IoTDBConnectionException, StatementExecutionException { + subs.dropTopic(""); + } + + @Test(expected = StatementExecutionException.class) // drop non-existent topic + public void testDropTopic_notCreate() + throws IoTDBConnectionException, StatementExecutionException { + subs.dropTopic("abab"); + } + + @Test(expected = StatementExecutionException.class) + public void testDropTopic_dup() throws IoTDBConnectionException, StatementExecutionException { + String dropName = "`topic-1*.`"; + subs.createTopic(dropName); + subs.dropTopic(dropName); + subs.dropTopic(dropName); + } + + @Test + public void testGetTopic_nonExist() throws IoTDBConnectionException, StatementExecutionException { + System.out.println(subs.getTopic("xxx")); + assertFalse(subs.getTopic("xxx").isPresent()); + } + + @Test + public void testGetTopic_exist() throws IoTDBConnectionException, StatementExecutionException { + subs.createTopic("exist_topic_name"); + assertTrue(subs.getTopic("exist_topic_name").isPresent()); + subs.dropTopic("exist_topic_name"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/autocommit/IoTDBTestAutoCommitFalseDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/autocommit/IoTDBTestAutoCommitFalseDataSetPullConsumerIT.java new file mode 100644 index 0000000000000..c2eededdfdeba --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/autocommit/IoTDBTestAutoCommitFalseDataSetPullConsumerIT.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.autocommit; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; + +/*** + * If autoCommit is set to false, not using commit to submit consumption progress will lead to repeated consumption; + * If set to true, it will not cause duplicate consumption (but considering the underlying implementation of the data subscription pipe is at-least-once semantics, there may also be cases of duplicate data) + * autoCommit itself is just an auxiliary feature for at least once semantics. The commit itself is only related to restart recovery. In normal situations without bugs (pure log/batch), whether to commit or not, it will still be sent once. + * Now the data subscription so-called progress information and the Pipe's progress information is a concept. + * For the semantics related to data subscription commit, here are some supplements: * + * In the implementation of data subscription, after a batch of messages is polled by the consumer, if they are not committed within a certain period of time (hardcoded value), the consumers under this subscription can repoll this batch of messages. This mechanism is designed to prevent messages from accumulating on the server side when the consumer does not explicitly commit with autoCommit set to false. However, this mechanism is not part of the functional definition (not exposed to the user), but only to handle some exceptional cases on the client side, such as when a consumer crashes after polling a batch of messages, or forgets to commit. + * Tests can be more focused on situations where autoCommit is true. In this case, if not explicitly committed, it is expected that all messages successfully polled by the consumer should be committed before the consumer is closed (reflected in the Pipe as no accumulated resources). + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTestAutoCommitFalseDataSetPullConsumerIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.TestAutoCommitFalseDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "Topic_auto_commit_false"; + private String pattern = device + ".**"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + private void consume_data_noCommit(SubscriptionTreePullConsumer consumer, Session session) + throws InterruptedException, + TException, + IOException, + StatementExecutionException, + IoTDBConnectionException { + while (true) { + Thread.sleep(1000); + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + System.out.println( + FORMAT.format(new Date()) + " consume data no commit:" + tablet.getRowSize()); + } + } + } + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_commit", "Auto_commit_FALSE", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + String sql1 = "select count(s_0) from " + device; + String sql2 = "select count(s_1) from " + device; + consume_data_noCommit(consumer, session_dest); + System.out.println(FORMAT.format(new Date()) + " src sql1: " + getCount(session_src, sql1)); + System.out.println("dest sql1: " + getCount(session_dest, sql1)); + System.out.println("dest2 sql1: " + getCount(session_dest2, sql1)); + check_count(4, sql1, "dest consume subscription before data:s_0"); + check_count(4, sql2, "dest consume subscription before data:s_1"); + check_count2(0, sql2, "dest2 consumption subscription previous data: s_1"); + + // Subscribe and then write data + insert_data(System.currentTimeMillis()); + consume_data_noCommit(consumer, session_dest2); + System.out.println("src sql1: " + getCount(session_src, sql1)); + System.out.println("dest sql1: " + getCount(session_dest, sql1)); + System.out.println("dest2 sql1: " + getCount(session_dest2, sql1)); + check_count(4, sql1, "dest consume subscription data 2:s_0"); + check_count2(4, sql1, "dest2 consumption subscription data 2:s_0"); + check_count2(4, sql2, "dest2 consumption subscription data 2:s_1"); + + // insert_data(1706659300000L); //2024-01-31 08:00:00+08:00 + // Will consume again + consume_data_noCommit(consumer, session_dest); + System.out.println("src sql1: " + getCount(session_src, sql1)); + System.out.println("dest sql1: " + getCount(session_dest, sql1)); + System.out.println("dest2 sql1: " + getCount(session_dest2, sql1)); + check_count(4, sql1, "dest consumption subscription before data3:s_0"); + check_count(4, sql2, "dest consume subscription before data3:s_1"); + check_count2(4, sql2, "dest2 consumption subscription before count 3 data:s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/autocommit/IoTDBTestAutoCommitTrueDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/autocommit/IoTDBTestAutoCommitTrueDataSetPullConsumerIT.java new file mode 100644 index 0000000000000..e8a45f0c1a4f2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/autocommit/IoTDBTestAutoCommitTrueDataSetPullConsumerIT.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.autocommit; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; + +/*** + * PullConsumer DataSet + * pattern: device + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTestAutoCommitTrueDataSetPullConsumerIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TestAutoCommitTrueDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_auto_commit_true"; + private String pattern = device + ".**"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } finally { + subs.dropTopic(topicName); + dropDB(database); + } + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + private void consume_data_noCommit(SubscriptionTreePullConsumer consumer, Session session) + throws InterruptedException, + TException, + IOException, + StatementExecutionException, + IoTDBConnectionException { + while (true) { + Thread.sleep(1000); + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + System.out.println( + FORMAT.format(new Date()) + " consume data no commit:" + tablet.getRowSize()); + } + } + } + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = "select count(s_0) from " + device; + consumer = create_pull_consumer("pull_commit", "Auto_commit_true", true, 1000L); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + System.out.println("First consumption"); + consume_data_noCommit(consumer, session_dest); + System.out.println(FORMAT.format(new Date()) + ", " + getCount(session_src, sql)); + check_count(4, sql, "Consumption subscription previous data: s_0"); + check_count(4, "select count(s_1) from " + device, "Consumption subscription before data: s_1"); + // Subscribe and then write data + insert_data(System.currentTimeMillis()); + consume_data_noCommit(consumer, session_dest2); + System.out.println("second consumption"); + check_count2(4, "select count(s_0) from " + device, "Consumption subscription data 2: s_0"); + check_count2(4, "select count(s_1) from " + device, "Consumption subscription data 2:s_1"); + // Consumed data will not be consumed again + consume_data_noCommit(consumer, session_dest); + System.out.println("Third consumption"); + check_count(4, "select count(s_0) from " + device, "Consumption subscription before data: s_0"); + check_count( + 4, "select count(s_1) from " + device, "Consumption subscription previous data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/format/IoTDBDBDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/format/IoTDBDBDataSetPullConsumerIT.java new file mode 100644 index 0000000000000..583338714b867 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/format/IoTDBDBDataSetPullConsumerIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer DataSet + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDBDataSetPullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_format_pull_dataset"; + private static final String pattern = database + ".**"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts") + .consumerGroupId("pull_mode_dataset") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + + String sql = "select count(s_0) from " + device; + System.out.println("src " + getCount(session_src, sql)); + System.out.println("dest " + getCount(session_dest, sql)); + check_count(8, sql, "consume after subscription"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest2); + System.out.println("src " + getCount(session_src, sql)); + System.out.println("dest " + getCount(session_dest, sql)); + System.out.println("dest2 " + getCount(session_dest2, sql)); + check_count2( + 12, sql, "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/format/IoTDBDBTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/format/IoTDBDBTsfilePullConsumerIT.java new file mode 100644 index 0000000000000..8cebc2110fc39 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/format/IoTDBDBTsfilePullConsumerIT.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer Tsfile + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDBTsfilePullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_format_pull_tsfile"; + private static final String pattern = database + ".**"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, true); + createDB(database); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db") + .consumerGroupId("mode_pull_tsfile") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + // Consumption data + List results = consume_tsfile_withFileCount(consumer, device); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 2, "number of received files"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + results = consume_tsfile_withFileCount(consumer, device); + assertEquals( + results.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals( + results.get(1), + 3, + "Number of received files: After unsubscribing and resubscribing, progress is not retained. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsDatasetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsDatasetPullConsumerIT.java new file mode 100644 index 0000000000000..514964784a274 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsDatasetPullConsumerIT.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pull consumer + * pattern: ts + * accurate time + * format: dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBAllTsDatasetPullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.AllTsDatasetPullConsumer"; + private static final String database2 = "root.test.AllTsDatasetPullConsumer"; + private static final String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private static final String pattern = device + ".s_0"; + private static final String topicName = "topic_loose_range_all_pull_dataset"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T23:59:59+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T23:59:59+08:00"; + consumer = create_pull_consumer("ts_accurate_dataset_pull", "loose_range_all", false, null); + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count_non_strict( + 3, "select count(s_0) from " + device, "Start time boundary data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Start time boundary data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Start time boundary data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Start time boundary data: s_1 " + device2); + + insert_data(System.currentTimeMillis(), device); // now + insert_data(System.currentTimeMillis(), device2); // now + System.out.println("src filter:" + getCount(session_src, sql)); + consume_data(consumer, session_dest); + check_count_non_strict( + 3, "select count(s_0) from " + device, "Write some real-time data later:s_0 " + device); + check_count( + 0, "select count(s_1) from " + device, "Write some real-time data later: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "not in range: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "not in range: s_1 " + device2); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + System.out.println("src filter:" + getCount(session_src, sql)); + check_count_non_strict( + 8, "select count(s_0) from " + device, "data within the time range: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "data within the time range: s_1 " + device); + check_count( + 0, "select count(s_0) from " + device2, "data within the time range: s_0 " + device2); + check_count( + 0, "select count(s_1) from " + device2, "data within the time range: s_1 " + device2); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + consume_data(consumer, session_dest); + check_count_non_strict( + 13, "select count(s_0) from " + device, "End time limit data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data: s_1" + device2); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1" + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1" + device2); + + consumer.unsubscribe(topicName); + consumer.subscribe(topicName); + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1" + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1" + device2); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsTsfilePullConsumerIT.java new file mode 100644 index 0000000000000..b1221990a933a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsTsfilePullConsumerIT.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/*** + * PullConsumer + * pattern: ts + * format: tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBAllTsTsfilePullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.AllTsTsfilePullConsumer"; + private static final String database2 = "root.AllTsTsfilePullConsumer"; + private static final String topicName = "TopicAllTsTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String pattern = device + ".s_0"; + private static final String device2 = database + ".d_1"; + private static SubscriptionTreePullConsumer consumer; + private List schemaList = new ArrayList<>(); + private long nowTimestamp; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + nowTimestamp = System.currentTimeMillis(); + createTopic_s( + topicName, + pattern, + null, + String.valueOf(nowTimestamp), + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_pattern_history_tsfile_pull") + .consumerGroupId("loose_range_all") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(nowTimestamp - 4000, device); + insert_data(nowTimestamp - 4000, device2); + System.out.println( + FORMAT.format(new Date()) + + " src filter:" + + getCount( + session_src, "select count(s_0) from " + device + " where time <=" + nowTimestamp)); + + // Consumption data + List paths = new ArrayList<>(3); + paths.add(device); + paths.add(device2); + paths.add(database2 + ".d_2"); + List rowCountList = consume_tsfile(consumer, paths); + assertGte(rowCountList.get(0), 8); + assertEquals(rowCountList.get(1), 0); + assertEquals(rowCountList.get(2), 0); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after cancellation"); + + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + rowCountList = consume_tsfile(consumer, paths); + assertGte( + rowCountList.get(0), + 13, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization."); + assertEquals(rowCountList.get(1), 0, "Unsubscribe and then resubscribe," + device2); + assertEquals(rowCountList.get(2), 0, "Unsubscribe and then resubscribe," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsfilePullConsumerSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsfilePullConsumerSnapshotIT.java new file mode 100644 index 0000000000000..e1ccad2513a10 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBAllTsfilePullConsumerSnapshotIT.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/*** + * PullConsumer + * pattern: ts + * format: tsfile + * loose-range: all + * mode: snapshot + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBAllTsfilePullConsumerSnapshotIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.AllTsfilePullConsumerSnapshot"; + private static final String database2 = "root.test.AllTsfilePullConsumerSnapshot"; + private static final String device = database + ".d_0"; + private static final String topicName = "TopicAllTsfilePullConsumerSnapshot"; + private static final String pattern = device + ".s_0"; + private static final String device2 = database + ".d_1"; + private static SubscriptionTreePullConsumer consumer; + private List schemaList = new ArrayList<>(); + private long nowTimestamp; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + nowTimestamp = System.currentTimeMillis(); + createTopic_s( + topicName, + pattern, + null, + String.valueOf(nowTimestamp), + true, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("snapshot_ts_pattern_tsfile_pull") + .consumerGroupId("loose_range_all") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(nowTimestamp - 4000, device); + insert_data(nowTimestamp - 4000, device2); + System.out.println( + FORMAT.format(new Date()) + + " src filter:" + + getCount( + session_src, "select count(s_0) from " + device + " where time <=" + nowTimestamp)); + + // Consumption data + List paths = new ArrayList<>(3); + paths.add(device); + paths.add(device2); + paths.add(database2 + ".d_2"); + List rowCountList = consume_tsfile(consumer, paths); + // Subscribe and write without consuming + assertEquals(rowCountList.get(0), 5, "Write without consume after subscription"); + assertEquals(rowCountList.get(1), 0); + assertEquals(rowCountList.get(2), 0); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + rowCountList = consume_tsfile(consumer, paths); + assertEquals( + rowCountList.get(0), + 10, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization."); + assertEquals(rowCountList.get(1), 0, "Unsubscribe and then resubscribe," + device2); + assertEquals(rowCountList.get(2), 0, "Unsubscribe and then resubscribe," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBPathDeviceDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBPathDeviceDataSetPullConsumerIT.java new file mode 100644 index 0000000000000..bbd845fd50701 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBPathDeviceDataSetPullConsumerIT.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pull consumer + * format: dataset + * loose-range: path + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBPathDeviceDataSetPullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.PathDeviceDataSetPullConsumer"; + private static final String database2 = "root.PathDeviceDataSetPullConsumer"; + private static final String topicName = "TopicPathDeviceDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private String pattern = device + ".**"; + private String device2 = database + ".d_1"; + private static SubscriptionTreePullConsumer consumer; + private List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + null, + "now", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = "select count(s_0) from " + device; + consumer = create_pull_consumer("device_pattern_dataset_pull", "loose_range_path", false, null); + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 30000L, device); + insert_data(System.currentTimeMillis() - 30000L, device2); + // Consumption data + consume_data(consumer, session_dest); + System.out.println("src: " + getCount(session_src, sql)); + check_count(10, sql, "Consumption data: s_0 " + device); + check_count(10, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + device2, "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(15, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(15, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + device2, "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBPathDeviceTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBPathDeviceTsfilePullConsumerIT.java new file mode 100644 index 0000000000000..e67deb8002727 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBPathDeviceTsfilePullConsumerIT.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * format: tsfile + * loose-range: time + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBPathDeviceTsfilePullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.PathDeviceTsfilePullConsumer"; + private static final String database2 = "root.PathDeviceTsfilePullConsumer"; + private static final String topicName = "TopicPathDeviceTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String pattern = device + ".**"; + private static final String device2 = database + ".d_1"; + private static SubscriptionTreePullConsumer consumer; + private List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + null, + null, + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("pull_path_device_tsfile") + .consumerGroupId("loose_range_path") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(database2 + ".d_2"); + List results = consume_tsfile(consumer, devices); + + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 0); + assertEquals(results.get(2), 0); + assertEquals(results.get(3), 2, "number of received files"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after cancellation"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + results = consume_tsfile(consumer, devices); + assertEquals( + results.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 0, "Unsubscribe and then subscribe again," + device2); + assertEquals(results.get(2), 0, "Subscribe again after cancellation," + database2 + ".d_2"); + assertEquals(results.get(3), 3, "Number of received files: resubscribe after unsubscribe"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBTimeTsDatasetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBTimeTsDatasetPullConsumerIT.java new file mode 100644 index 0000000000000..e5964833cb31c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBTimeTsDatasetPullConsumerIT.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pull consumer + * pattern: ts + * loose-range: time + * accurate time range + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeTsDatasetPullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TimeTsDatasetPullConsumer"; + private static final String database2 = "root.TimeTsDatasetPullConsumer"; + private static final String topicName = "TopicTimeTsDatasetPullConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String device3 = database2 + ".d_2"; + private String pattern = device + ".s_0"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T23:59:59+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + device3 + "(time,s_0,s_1)values(1000,132,4567.89);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T23:59:59+08:00"; + consumer = + create_pull_consumer("pull_ts_pattern_accurate_dataset", "loose_range_time", false, null); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count_non_strict(3, sql, "Start time boundary data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Start time boundary data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Start time boundary data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Start time limit data: s_1 " + device2); + + insert_data(System.currentTimeMillis(), device); // not in range + insert_data(System.currentTimeMillis(), device2); // not in range + System.out.println("src filter:" + getCount(session_src, sql)); + + consume_data(consumer, session_dest); + check_count_non_strict(3, sql, "After writing some real-time data: s_0 " + device); + check_count( + 0, "select count(s_1) from " + device, "Write some real-time data later:s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "not in range: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "not in range: s_1 " + device2); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + System.out.println("src filter:" + getCount(session_src, sql)); + + check_count_non_strict(8, sql, "Data within time range: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "data within the time range: s_1 " + device); + check_count( + 0, "select count(s_0) from " + device2, "Data within the time range: s_0 " + device2); + check_count( + 0, "select count(s_1) from " + device2, "Data within the time range: s_1 " + device2); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + + consume_data(consumer, session_dest); + check_count_non_strict(13, sql, "End time limit data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data: s_1 " + device2); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1 " + device2); + + consumer.unsubscribe(topicName); + consumer.subscribe(topicName); + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1 " + device2); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBTimeTsTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBTimeTsTsfilePullConsumerIT.java new file mode 100644 index 0000000000000..1a55940c14d47 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/loose_range/IoTDBTimeTsTsfilePullConsumerIT.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: ts + * format: tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeTsTsfilePullConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TimeTsTsfilePullConsumer"; + private static final String database2 = "root.TimeTsTsfilePullConsumer"; + private static final String topicName = "TopicTimeTsTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String pattern = device + ".s_0"; + private static final String device2 = database + ".d_1"; + private static final String device3 = database2 + ".d_2"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + private static List rowCountList; + private long nowTimestamp; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + nowTimestamp = System.currentTimeMillis(); + createTopic_s( + topicName, + pattern, + null, + String.valueOf(nowTimestamp), + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + device3 + "(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_pattern_tsfile_pull") + .consumerGroupId("loose_range_time") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + insert_data(nowTimestamp - 4000, device); + insert_data(nowTimestamp - 4000, device2); + + String sql = "select count(s_0) from " + device + " where time <=" + nowTimestamp; + System.out.println("TimeTsTsfilePullConsumer src1 filter:" + getCount(session_src, sql)); + + // Consumption data + List paths = new ArrayList<>(3); + paths.add(device); + paths.add(device2); + paths.add(device3); + + rowCountList = consume_tsfile(consumer, paths); + assertGte(rowCountList.get(0), 8, device); + assertEquals(rowCountList.get(1), 0, device2); + assertEquals(rowCountList.get(2), 0, device3); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + System.out.println("TimeTsTsfilePullConsumer src2 filter:" + getCount(session_src, sql)); + + rowCountList = consume_tsfile(consumer, paths); + assertGte( + rowCountList.get(0), + 13, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization. Actual=" + + rowCountList.get(0)); + assertEquals(rowCountList.get(1), 0, "Unsubscribe and subscribe again," + device2); + assertEquals(rowCountList.get(2), 0, "Unsubscribe and then resubscribe," + device3); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerDataSetIT.java new file mode 100644 index 0000000000000..7ee15fdcbd1b2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerDataSetIT.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBSnapshotDevicePullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.SnapshotDevicePullConsumerDataSet"; + private static final String database2 = "root.SnapshotDevicePullConsumerDataSet"; + private static final String topicName = "topicSnapshotDevicePullConsumerDataSet"; + private String device = database + ".d_0"; + + private String pattern = device + ".**"; + private static SubscriptionTreePullConsumer consumer; + private List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false, TopicConstant.MODE_SNAPSHOT_VALUE, null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_mode", "device_snapshot_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + Thread.sleep(1000); + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from " + device; + System.out.println("src: " + getCount(session_src, sql)); + check_count(4, sql, "Consumption data:" + pattern); + check_count(4, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerTsfileIT.java new file mode 100644 index 0000000000000..b865eb8282edc --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerTsfileIT.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBSnapshotDevicePullConsumerTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.SnapshotDevicePullConsumerTsfile"; + private static final String database2 = "root.SnapshotDevicePullConsumerTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicSnapshotDevicePullConsumerTsfile"; + private static final String pattern = device + ".**"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true, TopicConstant.MODE_SNAPSHOT_VALUE, null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_tsfile_snapshot") + .consumerGroupId("pull_mode") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(System.currentTimeMillis()); + System.out.println("src :" + getCount(session_src, "select count(s_0) from " + device)); + + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(database + ".d_1"); + devices.add(database2 + ".d_2"); + List rowCounts = consume_tsfile(consumer, devices); + + assertEquals(rowCounts.get(0), 5); + assertEquals(rowCounts.get(1), 0); + assertEquals(rowCounts.get(2), 0); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + // Consumption data: Progress is not retained after unsubscribing and then re-subscribing. Full + // synchronization. + rowCounts = consume_tsfile(consumer, devices); + assertEquals( + rowCounts.get(0), + 10, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals(rowCounts.get(1), 0, "Unsubscribe and resubscribe," + database + ".d_1"); + assertEquals( + rowCounts.get(2), 0, "Cancel subscription and subscribe again," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java new file mode 100644 index 0000000000000..05b061f733c3d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBConsumer2With1TopicShareProcessDataSetIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicConsumer2With1TopicShareProcessDataSet"; + private String pattern = device + ".**"; + private SubscriptionTreePullConsumer consumer2; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("g1", "c1", false, null); + consumer2 = create_pull_consumer("g1", "c2", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + assertEquals(subs.getSubscriptions().size(), 0, "Before subscribing, show subscriptions"); + consumer.subscribe(topicName); + consumer2.subscribe(topicName); + System.out.println("After subscription:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + consume_data(consumer, session_dest); + check_count( + 5, "select count(s_0) from " + device, "Consumption subscription before data: s_0 "); + check_count( + 5, "select count(s_1) from " + device, "Consumption subscription before data: s_1 "); + // Subscribe and then write data + insert_data(System.currentTimeMillis()); + consume_data(consumer2, session_dest2); + check_count2(5, "select count(s_0) from " + device, "Consumption subscription data: s_0"); + check_count2(5, "select count(s_1) from " + device, "Consumption subscription data: s_1"); + + // Consumed data will not be consumed again + consume_data(consumer, session_dest); + check_count(5, "select count(s_0) from " + device, "Consumption subscription before data: s_0"); + check_count( + 5, "select count(s_1) from " + device, "Consumption subscription previous data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java new file mode 100644 index 0000000000000..bcbf6d12ed358 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * format: tsfile + * pattern:device + * Same group pull consumer share progress + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBConsumer2With1TopicShareProcessTsfileIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicConsumer2With1TopicShareProcessTsfile"; + private static List schemaList = new ArrayList<>(); + private String pattern = device + ".**"; + private SubscriptionTreePullConsumer consumer2; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, true); + createDB(database); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("tsfile_group_share_process", "c1", false, null); + consumer2 = create_pull_consumer("tsfile_group_share_process", "c2", false, 10L); + // Subscribe + assertEquals( + subs.getSubscriptions(topicName).size(), 0, "Show subscriptions before subscribing"); + consumer.subscribe(topicName); + consumer2.subscribe(topicName); + subs.getSubscriptions(topicName).forEach((System.out::println)); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + // insert 1000 records + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + Thread thread1 = + new Thread( + () -> { + try { + rowCount1.addAndGet(consume_tsfile(consumer, device)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + Thread thread2 = + new Thread( + () -> { + try { + rowCount2.addAndGet(consume_tsfile(consumer2, device)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + thread1.start(); + thread2.start(); + thread.start(); + thread1.join(); + thread2.join(); + thread.join(); + + System.out.println("src=" + getCount(session_src, "select count(s_0) from " + device)); + System.out.println("rowCount1=" + rowCount1.get()); + System.out.println("rowCount2=" + rowCount2.get()); + AWAIT.untilAsserted( + () -> + assertGte( + rowCount1.get() + rowCount2.get(), + getCount(session_src, "select count(s_0) from " + device), + "consumer share process rowCount1=" + + rowCount1.get() + + " rowCount2=" + + rowCount2.get() + + " src=" + + getCount(session_src, "select count(s_0) from " + device))); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java new file mode 100644 index 0000000000000..03dae24d398c8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBMultiGroupVsMultiConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.pullMultiGroupVsMultiConsumer"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + private String topicNamePrefix = "TopicPullMultiGroupVsMultiConsumer_"; + private int tsCount = 10; + private int consumertCount = 10; + private List consumers = new ArrayList<>(consumertCount); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + for (int i = 0; i < tsCount; i++) { + createTopic_s(topicNamePrefix + i, device + ".s_" + i, null, null, false); + session_src.createTimeseries( + device + ".s_" + i, TSDataType.INT32, TSEncoding.RLE, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_" + i, TSDataType.INT32, TSEncoding.RLE, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_" + i, TSDataType.INT32, TSEncoding.RLE, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_" + i, TSDataType.INT32)); + } + System.out.println("topics:" + subs.getTopics()); + } + + @Override + @After + public void tearDown() throws Exception { + for (SubscriptionTreePullConsumer c : consumers) { + try { + c.close(); + } catch (Exception e) { + } + } + for (int i = 0; i < tsCount; i++) { + subs.dropTopic(topicNamePrefix + i); + } + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + for (int i = 0; i < tsCount; i++) { + tablet.addValue( + schemaList.get(i).getMeasurementName(), rowIndex, (row + 1) * 20 + i * 1000 + row); + } + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + /*** + * |c0|t0|g1| + * |c1|t0|g1| + * |c2|t1|g1| + * |c3|t1|g1| + * |c4|t2,t3|g2| + * |c5|t3,t4|g2| + * |c6|t2,t4|g2| + * |c7|t0,t3|g3| + * |c8|t6|g3| + * |c9|t0,t3|g3| + */ + @Test + public void do_test() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + int i = 0; + for (i = 0; i < 4; i++) { + consumers.add( + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_" + i) + .consumerGroupId("pull_group_id_1") + .buildPullConsumer()); + } + for (; i < 7; i++) { + consumers.add( + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_" + i) + .consumerGroupId("pull_group_id_2") + .buildPullConsumer()); + } + for (; i < consumertCount; i++) { + consumers.add( + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_" + i) + .consumerGroupId("pull_group_id_3") + .buildPullConsumer()); + } + for (int j = 0; j < consumertCount; j++) { + consumers.get(j).open(); + } + + consumers.get(0).subscribe(topicNamePrefix + 0); + consumers.get(1).subscribe(topicNamePrefix + 0); + consumers.get(2).subscribe(topicNamePrefix + 1); + consumers.get(3).subscribe(topicNamePrefix + 1); + consumers.get(4).subscribe(topicNamePrefix + 2, topicNamePrefix + 3); + consumers.get(5).subscribe(topicNamePrefix + 3, topicNamePrefix + 4); + consumers.get(6).subscribe(topicNamePrefix + 2, topicNamePrefix + 4); + consumers.get(7).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + consumers.get(8).subscribe(topicNamePrefix + 6); + consumers.get(9).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + + subs.getSubscriptions().forEach(System.out::println); + // Write data + System.out.println("Write data 1"); + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consume_data(consumers.get(0), session_dest); + System.out.println("src:" + getCount(session_src, "select count(s_0) from " + device)); + check_count(5, "select count(s_0) from " + device, "1 post-consumption check: s_0"); + for (i = 1; i < tsCount; i++) { + check_count(0, "select count(s_" + i + ") from " + device, "1 pre-consumption check: s_" + i); + } + consume_data(consumers.get(2), session_dest); + check_count(5, "select count(s_1) from " + device, "1 post-consumption check: s_1"); + consume_data(consumers.get(4), session_dest); + check_count(5, "select count(s_2) from " + device, "1 post-consumption check: s_2"); + check_count(5, "select count(s_3) from " + device, "1 post-consumption check: s_3"); + for (i = 4; i < tsCount; i++) { + check_count(0, "select count(s_" + i + ") from " + device, "1 pre-consumption check: s_" + i); + } + + System.out.println("Write data 2"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src:" + getCount(session_src, "select count(s_0) from " + device)); + consume_data(consumers.get(1), session_dest2); + consume_data(consumers.get(3), session_dest2); + consume_data(consumers.get(5), session_dest2); + consume_data(consumers.get(8), session_dest2); + + for (i = 0; i < 4; i++) { + if (i == 2) continue; + check_count2( + 5, "select count(s_" + i + ") from " + device, "2 pre-consumption check: s_" + i); + } + check_count2(10, "select count(s_4) from " + device, "2 check dest2:s_4 after consumption"); + check_count2(10, "select count(s_6) from " + device, "2 check dest2:s_6 after consumption"); + + consume_data(consumers.get(7), session_dest); + check_count(10, "select count(s_0) from " + device, "2 post-consumption check: s_0"); + check_count(10, "select count(s_3) from " + device, "2 post-consumption check: s_3"); + + System.out.println("Write data 3"); + insert_data(System.currentTimeMillis()); + consume_data(consumers.get(7), session_dest2); + check_count2(10, "select count(s_0) from " + device, "3 check dest2:s_0 after consumption"); + check_count2(10, "select count(s_3) from " + device, "3 consume check dest2:s_3"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java new file mode 100644 index 0000000000000..88a810e18ce5f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: fixed time range + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBOneConsumerMultiTopicsDatasetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsDataset"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + + private String pattern = device + ".s_0"; + private String pattern2 = device + ".s_1"; + private String topicName = "topic1_OneConsumerMultiTopicsDataset"; + private String topicName2 = "topic2_OneConsumerMultiTopicsDataset"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-01T00:00:00+08:00", false); + createTopic_s( + topicName2, pattern2, "2024-01-01T00:00:00+08:00", "2024-03-13T00:00:00+08:00", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + Thread.sleep(1000); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,6.78);"); // 2024-03-13 08:00:00+08:00 + // Subscribe + consumer = create_pull_consumer("multi_1consumer_multiTopics", "c1", false, null); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + + consumer.subscribe(topicName, topicName2); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "show subscriptions after subscription"); + + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + insert_data(System.currentTimeMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + thread.start(); + thread.join(); + String sql1 = "select count(s_0) from " + device; + String sql2 = "select count(s_1) from " + device; + System.out.println("src s_0:" + getCount(session_src, sql1)); + System.out.println("src s_1:" + getCount(session_src, sql2)); + // Consumption data + consume_data(consumer, session_dest); + System.out.println("dest s_0:" + getCount(session_dest, sql1)); + System.out.println("dest s_1:" + getCount(session_dest, sql2)); + AWAIT.untilAsserted( + () -> { + check_count(5, sql1, "Consumption data:" + pattern); + check_count(5, sql2, "Consumption data:" + pattern2); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Subscribe and then write data + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,1231,321.45);"); // 2023-12-31 08:00:00+08:00 + + // Consumption data + consume_data(consumer, session_dest); + AWAIT.untilAsserted( + () -> { + check_count(5, sql1, "consume data again:" + pattern); + check_count(10, sql2, "consume data again:" + pattern2); + }); + // Unsubscribe + consumer.unsubscribe(topicName, topicName2); + System.out.println("###### Query after unsubscription again:"); + subs.getSubscriptions().forEach((System.out::println)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java new file mode 100644 index 0000000000000..e19f0908ad5b5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: Historical data + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBOneConsumerMultiTopicsMixIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsMix"; + private static final String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String pattern2 = "root.**"; + private String topicName = "topic1_OneConsumerMultiTopicsMix"; + private String topicName2 = "topic2_OneConsumerMultiTopicsMix"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "now", false); + createTopic_s(topicName2, pattern2, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + assertTrue(subs.getTopic(topicName2).isPresent(), "Create show topics 2"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + // Subscribe + consumer = create_pull_consumer("multi_1consumer_mix", "tsfile_dataset", false, null); + System.out.println("###### Before Subscription Query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName, topicName2); + long timestamp = System.currentTimeMillis(); + System.out.println("###### Subscription Query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "show subscriptions after subscription"); + // Subscribe and then write data + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + insert_data(System.currentTimeMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + thread.start(); + + AtomicInteger rowCount = new AtomicInteger(0); + Thread thread2 = + new Thread( + () -> { + while (true) { + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final Iterator it = + message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + try { + session_dest.insertTablet(tablet); + // + // System.out.println(format.format(new Date())+" "+tablet.rowSize); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_1", true)), + null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ".s_1:" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + consumer.commitSync(messages); + break; + } + } + } + } + }); + Thread thread3 = + new Thread( + () -> { + while (true) { + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final Iterator it = + message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + try { + session_dest.insertTablet(tablet); + System.out.println( + FORMAT.format(new Date()) + " " + tablet.getRowSize()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_1", true)), + null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ":" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + consumer.commitSync(messages); + break; + } + } + } + } + }); + thread2.start(); + thread.join(); + thread2.join(5000); + String sql1 = "select count(s_0) from " + device + " where time <= " + timestamp; + String sql2 = "select count(s_1) from " + device + " where time <= " + timestamp; + + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql1)); + AWAIT.untilAsserted( + () -> { + // Consumption data + check_count(6, sql1, "Consumption data:" + pattern); + check_count(0, sql2, "Consumption data:" + pattern2); + assertEquals(rowCount.get(), 6, "tsfile consumer"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + System.out.println("###### After unsubscribing query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "Unsubscribe 1 and then show subscriptions"); + + // Unsubscribe and then write data + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,3.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + session_src.executeNonQueryStatement("flush"); + System.out.println( + FORMAT.format(new Date()) + + " Unsubscribe after writing data src:" + + getCount(session_src, sql1)); + + thread3.start(); + thread3.join(5000); + System.out.println(FORMAT.format(new Date())); + AWAIT.untilAsserted( + () -> { + assertEquals( + rowCount.get(), 12, "Re-consume data: tsfile consumer " + FORMAT.format(new Date())); + check_count(6, sql1, "consume data again:" + pattern); + check_count(0, sql2, "Reconsume data:" + pattern2); + }); + // close + consumer.close(); + try { + consumer.subscribe(topicName, topicName2); + } catch (Exception e) { + System.out.println("subscribe again after close, expecting an exception"); + } + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after close"); + subs.getSubscriptions().forEach((System.out::println)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java new file mode 100644 index 0000000000000..a9fecef2dc90c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/*** + * 1 consumer subscribes to 2 topics: historical data + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBOneConsumerMultiTopicsTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsTsfile"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + private String pattern = database + ".**"; + private String database2 = "root.OneConsumerMultiTopicsTsfile"; + private String pattern2 = database2 + ".**"; + private String device2 = database2 + ".d_0"; + private String topicName = "topic1_OneConsumerMultiTopicsTsfile"; + private String topicName2 = "topic2_OneConsumerMultiTopicsTsfile"; + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, "now", null, true); + createTopic_s(topicName2, pattern2, "now", null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + assertTrue(subs.getTopic(topicName2).isPresent(), "Create show topics 2"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + Thread.sleep(1000); + // Write data before subscribing + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer = create_pull_consumer("multi_tsfile_2topic", "1_consumer", false, null); + System.out.println("###### Subscription Query Before:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName, topicName2); + System.out.println("###### Subscribe and query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "subscribe then show subscriptions"); + + final AtomicInteger rowCount = new AtomicInteger(); + Thread thread1 = + new Thread( + () -> { + List devices = new ArrayList<>(2); + devices.add(device); + devices.add(device2); + try { + List results = consume_tsfile(consumer, devices); + System.out.println(results); + rowCount.addAndGet(results.get(0)); + rowCount.addAndGet(results.get(1)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + thread1.start(); + // Subscribe and then write data + Thread thread = + new Thread( + () -> { + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp, device); + insert_data(timestamp, device2); + timestamp += 30000; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + thread.start(); + thread.join(); + thread1.join(); + + System.out.println( + "src insert " + device + " :" + getCount(session_src, "select count(s_0) from " + device)); + System.out.println( + "src insert " + + device2 + + " :" + + getCount(session_src, "select count(s_0) from " + device2)); + assertEquals(rowCount.get(), 200, "After first consumption"); + // Unsubscribe + consumer.unsubscribe(topicName); + System.out.println("###### After cancellation query:"); + subs.getSubscriptions(topicName).forEach((System.out::println)); + assertEquals( + subs.getSubscriptions(topicName).size(), 0, "Unsubscribe 1 and then show subscriptions"); + + // Unsubscribe and then write data + insert_data(System.currentTimeMillis(), device2); + int result = consume_tsfile(consumer, device2); + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,3.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + assertEquals(result, 5, "After the second consumption"); + + // close + consumer.close(); + try { + consumer.subscribe(topicName, topicName2); + } catch (Exception e) { + System.out.println("subscribe again after close, expecting an exception"); + } + assertEquals(subs.getSubscriptions(topicName).size(), 0, "show subscriptions after close"); + subs.getSubscriptions(topicName).forEach((System.out::println)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumeTsfileIT.java new file mode 100644 index 0000000000000..b2eee482e8121 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumeTsfileIT.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: db + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDBPatternPullConsumeTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBPatternPullConsumeTsfile"; + private static final String database2 = "root.DBPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicDBPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".**"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + // Consumption data + List results = consume_tsfile_withFileCount(consumer, device); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 3, "number of files received"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 0, "Show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + results = consume_tsfile_withFileCount(consumer, device); + assertEquals( + results.get(0), + 15, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 4, "Number of files received: resubscribe after unsubscribe"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..d403c0005f99a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumerDataSetIT.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * format: dataset + * pattern:db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDBPatternPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBPatternPullConsumerDataSet"; + private static final String database2 = "root.DBPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicDBPatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + private String pattern = database + ".**"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("db_dataset_snapshot", "pull_mode", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption Data: s_0"); + check_count(8, "select count(s_1) from " + device, "Consumption Data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDefaultPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDefaultPatternPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..271755144e1a8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDefaultPatternPullConsumerDataSetIT.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDefaultPatternPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DefaultPatternPullConsumerDataSet"; + private static final String database2 = "root.DefaultPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicDefaultPatternPullConsumerDataSet"; + private static SubscriptionTreePullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, null, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_pattern", "default_pattern_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption data: s_0"); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(1, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumeTsfileIT.java new file mode 100644 index 0000000000000..398147e9e00d9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumeTsfileIT.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDevicePatternPullConsumeTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DevicePatternPullConsumeTsfile"; + private static final String database2 = "root.DevicePatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String device3 = database2 + ".d_2"; + private static final String topicName = "topicDevicePatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".**"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_pattern_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(device3); + List results = consume_tsfile(consumer, devices); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 0); + assertEquals(results.get(2), 0); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscribe"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + results = consume_tsfile(consumer, devices); + assertEquals( + results.get(0), + 15, + "After unsubscribing and resubscribing, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 0, "Cancel subscription and subscribe again," + device2); + assertEquals(results.get(2), 0, "Unsubscribe and subscribe again," + device3); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..2cad94fc4ac73 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumerDataSetIT.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDevicePatternPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DevicePatternPullConsumerDataSet"; + private static final String database2 = "root.DevicePatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String device2 = database2 + ".d_2"; + private static final String topicName = "topicDevicePatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private String pattern = device + ".**"; + public static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device2 + ".s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + device2 + ".s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + device2 + ".s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + device2 + ".s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } finally { + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + } + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_pattern", "device_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from " + device; + System.out.println("src: " + getCount(session_src, sql)); + check_count(8, sql, "Consumption data:" + pattern); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + device2, "Consumption data:d_2"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained after unsubscribing and then re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatch2PatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatch2PatternPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..7517ae5ba8636 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatch2PatternPullConsumerDataSetIT.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pattern: root.*.d_*.* + * format: dataset + * time-range: history + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBMiddleMatch2PatternPullConsumerDataSetIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.MiddleMatch2PatternPullConsumerDataSet"; + private static final String database2 = "root.MiddleMatch2PatternPullConsumerDataSet"; + private static List devices = new ArrayList<>(3); + private static final String device = database + ".d_0"; + private static final String device2 = database2 + ".sd_1"; + private static final String device3 = database2 + ".d_2"; + private static final String topicName = "topicMiddleMatch2PatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private String pattern = "root.*.d_*.*"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false); + devices.add(device); + devices.add(device2); + devices.add(device3); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int64;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int64;"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = + create_pull_consumer("pull_pattern", "MiddleMatchPatternHistory_DataSet", false, null); + // Write data before subscribing + for (int i = 0; i < 3; i++) { + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + for (int i = 0; i < 3; i++) { + insert_data(System.currentTimeMillis() - 30000L, devices.get(i)); + } + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from "; + for (int i = 0; i < 3; i++) { + System.out.println( + "src " + devices.get(i) + ": " + getCount(session_src, sql + devices.get(i))); + } + check_count(0, sql + device, "Consumption data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(10, "select count(s_0) from " + device3, "Consumption data: s_0 " + device3); + check_count(10, "select count(s_1) from " + device3, "Consumption data: s_1 " + device3); + for (int i = 0; i < 3; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + } + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + for (int i = 0; i < 3; i++) { + System.out.println( + "src " + devices.get(i) + ": " + getCount(session_src, sql + devices.get(i))); + } + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(0, "select count(s_0) from " + device, "consume data again:" + device); + check_count(0, "select count(s_0) from " + device2, "consume data again:" + device2); + check_count(10, "select count(s_1) from " + device3, "Consumption data:" + device3); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumeTsfileIT.java new file mode 100644 index 0000000000000..2572c41de9369 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumeTsfileIT.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBMiddleMatchPatternPullConsumeTsfileIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.MiddleMatchPatternPullConsumeTsfile"; + private static final String database2 = "root.MiddleMatchPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicMiddleMatchPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = "root.**.d_*.**"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("MiddleMatchPatternHistory_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 20000); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(database2 + ".d_2"); + + List rowCounts = consume_tsfile(consumer, devices); + assertEquals(rowCounts.get(0), 10); + assertEquals(rowCounts.get(1), 1); + assertEquals(rowCounts.get(2), 1); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after cancellation"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + rowCounts = consume_tsfile(consumer, devices); + + assertEquals( + rowCounts.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals( + rowCounts.get(1), 1, "Cancel subscription and subscribe again," + database + ".d_1"); + assertEquals(rowCounts.get(2), 1, "Unsubscribe and resubscribe," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..9fb2c072b06a0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumerDataSetIT.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBMiddleMatchPatternPullConsumerDataSetIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.MiddleMatchPatternPullConsumerDataSet"; + private static final String database2 = "root.MiddleMatchPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicMiddleMatchPatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private String pattern = "root.**.d_*.s_0"; + public static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = + create_pull_consumer("pull_pattern", "MiddleMatchPatternHistory_DataSet", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from " + device; + System.out.println("src " + database + ".d_0.s_0: " + getCount(session_src, sql)); + System.out.println( + "src " + + database + + ".d_0.s_1: " + + getCount(session_src, "select count(s_1) from " + device)); + System.out.println( + "src " + + database + + ".d_1.s_0: " + + getCount(session_src, "select count(s_0) from " + database + ".d_1")); + System.out.println( + "src " + + database2 + + ".d_2.s_0: " + + getCount(session_src, "select count(s_0) from " + database2 + ".d_2")); + System.out.println("dest " + database + ".d_0.s_0: " + getCount(session_dest, sql)); + System.out.println( + "dest " + + database + + ".d_0.s_1: " + + getCount(session_dest, "select count(s_1) from " + device)); + System.out.println( + "dest " + + database + + ".d_1.s_0: " + + getCount(session_dest, "select count(s_0) from " + database + ".d_1")); + System.out.println( + "dest " + + database2 + + ".s_0: " + + getCount(session_dest, "select count(s_0) from " + database2 + ".d_2")); + check_count(10, sql, "Consumption Data:s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption Data:d_1"); + check_count(1, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + // Consumption data: Progress is not preserved if you unsubscribe and then resubscribe. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(15, "select count(s_0) from " + device, "Consume data again:s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBRootPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBRootPatternPullConsumeTsfileIT.java new file mode 100644 index 0000000000000..7c88dec3f806b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBRootPatternPullConsumeTsfileIT.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: db + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBRootPatternPullConsumeTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.RootPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicRootPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = "root.**"; + public static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + // Consumption data + List results = consume_tsfile_withFileCount(consumer, device); + assertEquals(results.get(0), 10, "Number of consumption data rows"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after unsubscribe"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + results = consume_tsfile_withFileCount(consumer, device); + assertEquals( + results.get(0), + 15, + "After unsubscribing and resubscribing, progress is not retained. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumeTsfileIT.java new file mode 100644 index 0000000000000..060d475d89b6f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumeTsfileIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: TS + * history + * tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTSPatternPullConsumeTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBPatternPullConsumeTsfile"; + private static final String database2 = "root.DBPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicDBPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + + public static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(database2 + ".d_2"); + List results = consume_tsfile(consumer, devices); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 0); + assertEquals(results.get(2), 0); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + results = consume_tsfile(consumer, devices); + + assertEquals( + results.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 0, "Subscribe again after unsubscribe," + database + ".d_1"); + assertEquals(results.get(2), 0, "Unsubscribe and then subscribe again," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..a47d53ccdb809 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumerDataSetIT.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTSPatternPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TSPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicTSPatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + public static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + Thread.sleep(1000); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_pattern", "ts_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption data: s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBAllPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBAllPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..c86a86373c419 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBAllPullConsumerDataSetIT.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBAllPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.AllPullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + // private String topicName = "`1-group.1-consumer.ts`"; + private String topicName = "topic_AllPullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_time_default_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + Thread.sleep(10000); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBHistoryPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBHistoryPullConsumerDataSetIT.java new file mode 100644 index 0000000000000..004b5928fc8d5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBHistoryPullConsumerDataSetIT.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBHistoryPullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.HistoryPullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + // private String topicName = "`1-group.1-consumer.ts`"; + private String topicName = "topic_HistoryPullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_history_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + + // System.out.println("Check consumption data"); + check_count(4, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + // System.out.println("After resubscribing:"); + // subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest2); + check_count2(8, "select count(s_0) from " + device, "consume data again:" + pattern); + check_count2(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBRealTimePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBRealTimePullConsumerDataSetIT.java new file mode 100644 index 0000000000000..25511d1b9e3e3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBRealTimePullConsumerDataSetIT.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBRealTimePullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.RealTimePullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "topic_RealTimePullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + createDB(database); + createTopic_s(topicName, pattern, "now", null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_realtime_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Consumption data + consume_data(consumer, session_dest); + check_count(0, "select count(s_0) from " + device, "Consumption data:" + pattern); + insert_data(System.currentTimeMillis()); + consume_data(consumer, session_dest); + check_count(4, "select count(s_0) from " + device, "Consumption data:" + pattern); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + insert_data(System.currentTimeMillis()); // now + consumer.subscribe(topicName); + System.out.println("After re-subscribing:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + // Consumption data + consume_data(consumer, session_dest2); + check_count(4, "select count(s_0) from " + device, "Consume data again:" + pattern); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBTimeRangeAccuratePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBTimeRangeAccuratePullConsumerDataSetIT.java new file mode 100644 index 0000000000000..fcfd216958338 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBTimeRangeAccuratePullConsumerDataSetIT.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeRangeAccuratePullConsumerDataSetIT + extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeRangeAccuratePullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "topic_TimeRangeAccuratePullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00", false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_accurate_dataset", false, null); + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + + insert_data(System.currentTimeMillis()); // now + consume_data(consumer, session_dest); + check_count( + 2, "select count(s_0) from " + device, "Write some real-time data after:" + pattern); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + check_count(6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + consume_data(consumer, session_dest); + check_count(10, "select count(s_0) from " + device, "End time boundary data:" + pattern); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + consume_data(consumer, session_dest); + check_count(11, "select count(s_0) from " + device, "End time limit data 2:" + pattern); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBTimeRangePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBTimeRangePullConsumerDataSetIT.java new file mode 100644 index 0000000000000..e490ea525690b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pullconsumer/time/IoTDBTimeRangePullConsumerDataSetIT.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Start time, end time are both closed intervals. If not specified, the time will be 00:00:00. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeRangePullConsumerDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeRangePullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "topic_TimeRangePullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T00:00:00+08:00", false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_time_range_dataset", false, null); + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + + insert_data(System.currentTimeMillis()); // now + consume_data(consumer, session_dest); + check_count( + 2, "select count(s_0) from " + device, "Write some real-time data later:" + pattern); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + check_count(6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + consume_data(consumer, session_dest); + // Because the end time is 2024-03-31 00:00:00, closed interval + check_count(8, "select count(s_0) from " + device, "End time limit data:" + pattern); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "End time boundary data:" + pattern); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeDataSetIT.java new file mode 100644 index 0000000000000..89cb0a11e43aa --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeDataSetIT.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer: BEFORE_CONSUME + * DataSet + * pattern: root.** + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTestPushConsumeDataSetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TestPushConsumeDataSet"; + private static final String topicName = "topic_TestPushConsumeDataSet"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = "root.**"; + private static SubscriptionTreePushConsumer consumer; + private static final String device = database + ".d_push_dataset"; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + final AtomicInteger rowCount = new AtomicInteger(0); + consumer = + new SubscriptionTreePushConsumer.Builder() + .nodeUrls(Collections.singletonList(SRC_HOST + ":" + SRC_PORT)) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_format") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/iotdb-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + String sql0 = "select count(s_0) from " + device; + String sql1 = "select count(s_1) from " + device; + long expectCount = getCount(session_src, sql0); + System.out.println("###### src " + expectCount); + long finalExpectCount = expectCount; + AWAIT.untilAsserted( + () -> { + assertEquals(getCount(session_dest, sql0), finalExpectCount, "First result:s_0"); + assertEquals(getCount(session_dest, sql1), finalExpectCount, "First result:s_1"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + expectCount = getCount(session_src, sql0); + System.out.println("###### src2 " + expectCount); + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + long finalExpectCount1 = expectCount; + AWAIT.untilAsserted( + () -> { + assertEquals(getCount(session_dest, sql0), finalExpectCount1, "Second result: s_0"); + assertEquals(getCount(session_dest, sql1), finalExpectCount1, "Second result: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeNoTargetDirTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeNoTargetDirTsfileIT.java new file mode 100644 index 0000000000000..a0ede5acca8f4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeNoTargetDirTsfileIT.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer:BEFORE_CONSUME + * TsFileHandler + * pattern: db + * target: no + * consumer_id: no + * group_id: no + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTestPushConsumeNoTargetDirTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TestPushConsumeNoTargetDirTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TestPushConsumeNoTargetDirTsfile"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = database + ".**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + System.out.println("### TestPushConsumeNoTargetDirTsfile ###"); + + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger rowCount = new AtomicInteger(0); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path = new Path(device, "s_0", true); + QueryDataSet dataset = + reader.query(QueryExpression.create(Collections.singletonList(path), null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + dataset.next(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceiveCount.get(), 2, "should 2 tsfile"); + assertEquals(rowCount.get(), 10, "should process 8 rows data"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + assertEquals(onReceiveCount.get(), 5, "should 5 tsfiles include 2 duplicates"); + assertEquals(rowCount.get(), 25, "should process 20 rows data, include "); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeTsfileIT.java new file mode 100644 index 0000000000000..b14a9bccf6972 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/format/IoTDBTestPushConsumeTsfileIT.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer:AFTER_CONSUME + * TsFileHandler + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTestPushConsumeTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TestPushConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TestPushConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = database + ".**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger rowCount = new AtomicInteger(0); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_TsFile_specify_target_dir_consumer") + .consumerGroupId("push_format") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path = new Path(device, "s_0", true); + QueryDataSet dataset = + reader.query(QueryExpression.create(Collections.singletonList(path), null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + String sql = "select count(s_0) from " + device; + System.out.println("src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount.get(), 8, "should process 8 rows data"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + AWAIT.untilAsserted( + () -> { + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + assertEquals(rowCount.get(), 20, "should process 12 rows data, include "); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..7a1b270da4528 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerIT.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * loose-range: all + * mode: live + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBLooseAllTsDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.LooseAllTsDatasetPushConsumer"; + private static final String database2 = "root.LooseAllTsDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_LooseAllTsDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + private String pattern = device + ".**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println("LooseAllTsDatasetPushConsumer src1: " + getCount(session_src, sql)); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_accurate_dataset_push_snapshot") + .consumerGroupId("loose_range_all") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(1706745600000L, device); // 2024-02-01 08:00:00+08:00 + insert_data(1706745600000L, device2); // 2024-02-01 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println("LooseAllTsDatasetPushConsumer src2: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 14, "select count(s_0) from " + device, "Consumption data: s_0 " + device); + check_count_non_strict( + 14, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println("LooseAllTsDatasetPushConsumer src3: " + getCount(session_src, sql)); + + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 16, "select count(s_0) from " + device, "consume data again: s_0 " + device); + check_count_non_strict( + 16, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerSnapshotIT.java new file mode 100644 index 0000000000000..dc4dac80a3c07 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerSnapshotIT.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * loose-range: all + * mode: snapshot + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBLooseAllTsDatasetPushConsumerSnapshotIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.LooseAllTsDatasetPushConsumerSnapshot"; + private static final String database2 = "root.LooseAllTsDatasetPushConsumerSnapshot"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_LooseAllTsDatasetPushConsumerSnapshot"; + private static List schemaList = new ArrayList<>(); + private String pattern = device + ".**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println("src: " + getCount(session_src, sql)); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_accurate_dataset_push_snapshot") + .consumerGroupId("loose_range_all") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(1706745600000L, device); // 2024-02-01 08:00:00+08:00 + insert_data(1706745600000L, device2); // 2024-02-01 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println("src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption data: s_0 " + device); + check_count_non_strict( + 9, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println("src: " + getCount(session_src, sql)); + + // Consumption data: Progress is not retained after cancellation and re-subscription. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "consume data again: s_0 " + device); + check_count_non_strict( + 11, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..16ff58f4e9c40 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBLooseAllTsfilePushConsumerIT.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * push consumer + * mode: live + * pattern: db + * loose-range: all + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBLooseAllTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.LooseAllTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = database + ".**"; + private String topicName = "topic_LooseAllTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + List rowCounts = new ArrayList<>(2); + rowCounts.add(new AtomicInteger(0)); + rowCounts.add(new AtomicInteger(0)); + final AtomicInteger onReceive = new AtomicInteger(0); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush"); + + List paths = new ArrayList<>(2); + paths.add(device); + paths.add(device2); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_ts_tsfile_push") + .consumerGroupId("loose_range_all") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 2; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(paths.get(i), "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } + System.out.println( + FORMAT.format(new Date()) + + " " + + rowCounts.get(0).get() + + "," + + rowCounts.get(1).get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 3); + assertGte(rowCounts.get(1).get(), 3); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 3); + assertGte(rowCounts.get(1).get(), 3); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 8); + assertGte(rowCounts.get(1).get(), 8); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10); + assertGte(rowCounts.get(1).get(), 10); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00, not in range + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10, "Inserted data is out of range"); + assertGte(rowCounts.get(1).get(), 10); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseDeviceTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseDeviceTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..dcd9065032e45 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseDeviceTsfilePushConsumerIT.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * loose range: path + * pattern: device + * push consumer + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBPathLooseDeviceTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.PathLooseDeviceTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = device + ".**"; + private String topicName = "topic_PathLooseDeviceTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger onReceive = new AtomicInteger(0); + List rowCounts = new ArrayList<>(2); + rowCounts.add(new AtomicInteger(0)); + rowCounts.add(new AtomicInteger(0)); + + List paths = new ArrayList<>(2); + paths.add(new Path(device, "s_0", true)); + paths.add(new Path(device2, "s_0", true)); + + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush"); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_device_tsfile_push") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 2; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1); + assertGte(rowCounts.get(0).get(), 3, "Write data before subscription" + device); + assertGte(rowCounts.get(0).get(), 3, "Write data before subscription" + device2); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1); + assertGte(rowCounts.get(0).get(), 3, "Write out-of-range data" + device); + assertGte(rowCounts.get(0).get(), 3, "Write out-of-range data" + device2); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 2); + assertGte(rowCounts.get(0).get(), 8, "write data" + device); + assertGte(rowCounts.get(0).get(), 8, "write data " + device2); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 3); + assertGte(rowCounts.get(0).get(), 10, "Write data: end boundary at " + device); + assertGte(rowCounts.get(0).get(), 10, "Write data: end boundary at " + device2); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 3); + assertGte(rowCounts.get(0).get(), 10, "Write data: > end " + device); + assertGte(rowCounts.get(0).get(), 10, "Write data: > end " + device2); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseTsDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseTsDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..c41386e8a3e06 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseTsDatasetPushConsumerIT.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * mode: DataSet + * pattern: ts + * loose-range: path + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBPathLooseTsDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.PathLooseTsDatasetPushConsumer"; + private static final String database2 = "root.PathLooseTsDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_PathLooseTsDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".d_0.s_0"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_accurate_dataset_consumer") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 4, + "select count(s_0) from " + device, + "Subscribe before writing data: s_0 " + device); + check_count( + 0, + "select count(s_1) from " + device, + "Subscribe before writing data: s_1 " + device); + check_count( + 0, + "select count(s_0) from " + device2, + "Subscribe before writing data: s_0 " + device2); + check_count( + 0, + "select count(s_1) from " + device2, + "Subscribe before writing data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + insert_data(System.currentTimeMillis(), device); // not in range + insert_data(System.currentTimeMillis(), device2); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Out-of-range data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Out-of-range data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Out-of-range data: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "Out-of-range data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + // Consumption data: Progress is not preserved if you unsubscribe and then resubscribe. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "consume data again:s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..a381ef02f548d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathLooseTsfilePushConsumerIT.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * loose range: path + * push consumer + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBPathLooseTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.PathLooseTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = database + ".**"; + private String topicName = "topic_PathLooseTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush"); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_db_tsfile_push") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 3); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 3); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 8); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 10); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 10); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathTsLooseDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathTsLooseDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..f9fbe32a7457d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBPathTsLooseDatasetPushConsumerIT.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBPathTsLooseDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.PathTsLooseDatasetPushConsumer"; + private static final String database2 = "root.PathTsLooseDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_PathTsLooseDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".d_0.s_0"; + public static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_accurate_dataset_consumer") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption Data: s_0" + device); + check_count_non_strict( + 0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "consume data again:s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..d298246e17e38 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsDatasetPushConsumerIT.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * loose-range: time + * live + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeLooseTsDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TimeLooseTsDatasetPushConsumer"; + private static final String database2 = "root.TimeLooseTsDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_TimeLooseTsDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = device + ".s_0"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + session_src.executeNonQueryStatement("flush"); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("push_dataset_ts_dataset_consumer") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "Consumption data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..26f4b15427482 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsTsfilePushConsumerIT.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * mode: live + * loose-range:path + * format: tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeLooseTsTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeLooseTsTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = device + ".s_0"; + private String topicName = "topic_TimeLooseTsTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger onReceive = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + for (int i = 0; i < 4; i++) { + rowCounts.add(new AtomicInteger(0)); + } + + List paths = new ArrayList<>(4); + paths.add(new Path(device, "s_0", true)); + paths.add(new Path(device, "s_1", true)); + paths.add(new Path(device2, "s_0", true)); + paths.add(new Path(device2, "s_1", true)); + + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_ts_tsfile_push") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + System.out.println( + FORMAT.format(new Date()) + " " + i + " " + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte( + rowCounts.get(0).get(), + 3, + device + ".s_0, subscribe before writing data start boundary"); + assertEquals( + rowCounts.get(1).get(), + 0, + device + ".s_1, Subscription before writing data start boundary"); + assertEquals( + rowCounts.get(2).get(), + 0, + device2 + ".s_0, Subscribe before writing data start boundary"); + assertEquals( + rowCounts.get(3).get(), + 0, + device2 + ".s_1, Subscription before writing data start boundary"); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 3, device + ".s_0, Write out-of-range data"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1, Write out-of-range data"); + assertEquals(rowCounts.get(2).get(), 0, device2 + ".s_0, Write out-of-range data"); + assertEquals(rowCounts.get(3).get(), 0, device2 + ".s_1, Write out-of-range data"); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 8, device + ".s_0, write normal data"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1, write normal data"); + assertEquals(rowCounts.get(2).get(), 0, device2 + ".s_0, Write normal data"); + assertEquals(rowCounts.get(3).get(), 0, device2 + ".s_1, Write normal data"); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10, device + ".s_0, write end boundary data"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1, write end boundary data"); + assertEquals(rowCounts.get(2).get(), 0, device2 + ".s_0, write end boundary data"); + assertEquals(rowCounts.get(3).get(), 0, device2 + ".s_1, write end boundary data"); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10, device + ".s_0, Write data outside end range"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_0, Write data outside of end range"); + assertEquals(rowCounts.get(2).get(), 0, device + ".s_0, Write data outside of end range"); + assertEquals(rowCounts.get(3).get(), 0, device + ".s_0, Write data outside of end range"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..3f9ed14b22f2d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeLooseTsfilePushConsumerIT.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeLooseTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeLooseTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = device + ".**"; + private String topicName = "topic_TimeLooseTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_db_tsfile_push") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + System.out.println("onReceive=" + onReceive.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1); + assertGte(rowCount.get(), 3); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1); + assertGte(rowCount.get(), 3); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 2); + assertGte(rowCount.get(), 8); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 3); + assertGte(rowCount.get(), 10); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + session_src.executeNonQueryStatement("flush"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 3); + assertGte(rowCount.get(), 10); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeTsLooseDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeTsLooseDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..3a9d1b24b746d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/loose_range/IoTDBTimeTsLooseDatasetPushConsumerIT.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * time loose + * live + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeTsLooseDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TimeTsLooseDatasetPushConsumer"; + private static final String database2 = "root.TimeTsLooseDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_TimeTsLooseDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + public static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } finally { + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + } + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("push_dataset_ts_dataset_consumer") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte( + getCount(session_dest, "select count(s_0) from " + device), + 9, + "Consumption data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush"); + + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + assertGte( + getCount(session_dest, "select count(s_0) from " + device), + 11, + "re-subscribing s_0 count=" + + getCount(session_dest, "select count(s_0) from " + device)); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/mode/IoTDBSnapshotTSPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/mode/IoTDBSnapshotTSPatternDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..d003cbceaedab --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/mode/IoTDBSnapshotTSPatternDatasetPushConsumerIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBSnapshotTSPatternDatasetPushConsumerIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.SnapshotTSPatternDatasetPushConsumer"; + private static final String database2 = "root.SnapshotTSPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_SnapshotTSPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = device + ".s_0"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false, TopicConstant.MODE_SNAPSHOT_VALUE, null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_dataset_snapshot") + .consumerGroupId("push_mode") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(4, "select count(s_0) from " + device, "Consumption data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "consume data again:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/mode/IoTDBSnapshotTSPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/mode/IoTDBSnapshotTSPatternTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..2b34045e463b1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/mode/IoTDBSnapshotTSPatternTsfilePushConsumerIT.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBSnapshotTSPatternTsfilePushConsumerIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.SnapshotTSPatternTsfilePushConsumer"; + private static final String database2 = "root.SnapshotTSPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_SnapshotTSPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = device + ".s_0"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_SNAPSHOT_VALUE, + null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + for (int i = 0; i < 4; i++) { + rowCounts.add(new AtomicInteger(0)); + } + + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path(database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_tsfile_snapshot") + .consumerGroupId("push_mode") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + System.out.println("onReceiveCount=" + onReceiveCount.get()); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + System.out.println( + FORMAT.format(new Date()) + + " rowCounts_" + + i + + ":" + + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + + insert_data(1707609600000L); // 2024-02-11 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceiveCount.get(), 1, "receive files"); + assertEquals(rowCounts.get(0).get(), 5, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceiveCount.get(), 2, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 15, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java new file mode 100644 index 0000000000000..0bca36eaa1b6a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: db + * Dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBConsumer2With1TopicShareProcessDataSetIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_Consumer2With1TopicShareProcessDataSet"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".**"; + private static SubscriptionTreePushConsumer consumer; + private static SubscriptionTreePushConsumer consumer2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + // insert_data(1706659200000L); + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_dataset_consumer_1") + .consumerGroupId("push_multi_test") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + consumer2 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_dataset_consumer_2") + .consumerGroupId("push_multi_test") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); + thread.start(); + + thread.join(); + System.out.println("After subscription:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + String sql = "select count(s_0) from " + device; + // The first 5 entries may have duplicate data + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertTrue(getCount(session_dest, sql) >= 0, "first consumer"); + assertTrue(getCount(session_dest2, sql) >= 0, "second Consumer"); + assertEquals( + getCount(session_dest, sql) + getCount(session_dest2, sql), + getCount(session_src, sql), + "share process"); + }); + System.out.println(FORMAT.format(new Date()) + " dest:" + getCount(session_dest, sql)); + System.out.println(FORMAT.format(new Date()) + " dest2:" + getCount(session_dest2, sql)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java new file mode 100644 index 0000000000000..92a8caa126b55 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: db + * tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBConsumer2With1TopicShareProcessTsfileIT + extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_Consumer2With1TopicShareProcessTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".**"; + private static SubscriptionTreePushConsumer consumer; + private static SubscriptionTreePushConsumer consumer2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile_consumer_1") + .consumerGroupId("push_multi_Consumer2With1TopicShareProcessTsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount1.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(next.getTimestamp()+","+next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + consumer2 = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile_consumer_2") + .consumerGroupId("push_multi_Consumer2With1TopicShareProcessTsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount2.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(next.getTimestamp()+","+next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); + thread.start(); + + thread.join(); + System.out.println("After subscription:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // The first 5 entries may have duplicate data + String sql = "select count(s_0) from " + device; + System.out.println("src " + getCount(session_src, sql)); + System.out.println("rowCount1.get()=" + rowCount1.get()); + System.out.println("rowCount2.get()=" + rowCount2.get()); + + AWAIT.untilAsserted( + () -> + assertGte( + rowCount1.get() + rowCount2.get(), getCount(session_src, sql), "share process")); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java new file mode 100644 index 0000000000000..17fa08cd9e2ce --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java @@ -0,0 +1,604 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * pattern: device, different db + * |c0|t0|g1| tsfile databasePrefix+"0.**", "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00" + * |c1|t0|g1| tsfile + * |c2|t1|g1| dataset(dest) databasePrefix+"1.**" + * |c3|t1|g1| dataset(dest2) + * |c4|t2,t3|g2| dataset(dest) databasePrefix+"2.**", "now", null; databasePrefix+"3.**", null, "now" + * |c5|t3,t4|g2| dataset(dest2) databasePrefix+"4.**", null, "2024-03-31T23:59:59+08:00" + * |c6|t2,t4|g2| dataset(dest) + * |c7|t0,t3|g3| dataset(dest)/tsfile + * |c8|t6|g3| tsfile databasePrefix+"6.**", "now", null, + * |c9|t0,t3|g3| dataset(dest2)/tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBMultiGroupVsMultiConsumerIT extends AbstractSubscriptionTreeRegressionIT { + + private String topicNamePrefix = "topic_pushMultiGroupVsMultiConsumer_"; + private String databasePrefix = "root.test.pushMultiGroupVsMultiConsumer_"; + private int tsCount = 10; + private int consumertCount = 10; + private static List schemaList = new ArrayList<>(); + + private List consumers = new ArrayList<>(consumertCount); + private AtomicInteger rowCount00 = new AtomicInteger(0); + private AtomicInteger rowCount10 = new AtomicInteger(0); + private AtomicInteger rowCount70 = new AtomicInteger(0); + private AtomicInteger rowCount90 = new AtomicInteger(0); + private AtomicInteger rowCount6 = new AtomicInteger(0); + private String sql1 = "select count(s_0) from " + databasePrefix + "1.d_0"; + private String sql2 = "select count(s_0) from " + databasePrefix + "2.d_0"; + private String sql3 = "select count(s_0) from " + databasePrefix + "3.d_0"; + private String sql4 = "select count(s_0) from " + databasePrefix + "4.d_0"; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + for (int i = 0; i < tsCount; i++) { + createDB(databasePrefix + i); + } + createTopic_s( + topicNamePrefix + 0, + databasePrefix + "0.**", + "2024-01-01T00:00:00+08:00", + "2024-03-31T23:59:59+08:00", + true); + createTopic_s(topicNamePrefix + 1, databasePrefix + "1.**", null, null, false); + + createTopic_s(topicNamePrefix + 2, databasePrefix + "2.**", "now", null, false); + createTopic_s(topicNamePrefix + 3, databasePrefix + "3.**", null, "now", false); + createTopic_s( + topicNamePrefix + 4, databasePrefix + "4.**", null, "2024-03-31T23:59:59+08:00", false); + + createTopic_s(topicNamePrefix + 6, databasePrefix + "6.**", "now", null, true); + + createTopic_s( + topicNamePrefix + 5, databasePrefix + "5.**", "2024-01-01T00:00:00+08:00", null, false); + createTopic_s( + topicNamePrefix + 7, databasePrefix + "7.**", null, "2024-03-31T23:59:59+08:00", true); + createTopic_s(topicNamePrefix + 8, databasePrefix + "8.**", null, "now", true); + createTopic_s( + topicNamePrefix + 9, databasePrefix + "9.**", "2024-01-01T00:00:00+08:00", null, true); + + subs.getTopics().forEach(System.out::println); + } + + @Override + @After + public void tearDown() throws Exception { + System.out.println(databasePrefix + "1.d_0:[src]" + getCount(session_src, sql1)); + System.out.println(databasePrefix + "1.d_0:[dest]" + getCount(session_dest, sql1)); + System.out.println(databasePrefix + "1.d_0:[dest2]" + getCount(session_dest2, sql1)); + System.out.println(databasePrefix + "2.d_0:[src]" + getCount(session_src, sql2)); + System.out.println(databasePrefix + "2.d_0:[dest]" + getCount(session_dest, sql2)); + System.out.println(databasePrefix + "2.d_0:[dest2]" + getCount(session_dest2, sql2)); + System.out.println(databasePrefix + "3.d_0:[src]" + getCount(session_src, sql3)); + System.out.println(databasePrefix + "3.d_0:[dest]" + getCount(session_dest, sql3)); + System.out.println(databasePrefix + "3.d_0:[dest2]" + getCount(session_dest2, sql3)); + System.out.println(databasePrefix + "4.d_0:[src]" + getCount(session_src, sql4)); + System.out.println(databasePrefix + "4.d_0:[dest]" + getCount(session_dest, sql4)); + System.out.println(databasePrefix + "4.d_0:[dest2]" + getCount(session_dest2, sql4)); + System.out.println("rowCount00.get()=" + rowCount00.get()); + System.out.println("rowCount10.get()=" + rowCount10.get()); + System.out.println("rowCount70.get()=" + rowCount70.get()); + System.out.println("rowCount90.get()=" + rowCount90.get()); + System.out.println("rowCount6.get()=" + rowCount6.get()); + for (SubscriptionTreePushConsumer c : consumers) { + try { + c.close(); + } catch (Exception e) { + } + } + for (int i = 0; i < tsCount; i++) { + subs.dropTopic(topicNamePrefix + i); + } + dropDB(databasePrefix); + super.tearDown(); + } + + private void insert_data(long timestamp, String device, int rows) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, rows); + int rowIndex = 0; + for (int row = 0; row < rows; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, (row + 1) * 1400 + row); + tablet.addValue(schemaList.get(1).getMeasurementName(), rowIndex, (row + 1) * 100 + 0.5); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + /*** + * |c0|t0|g1| tsfile databasePrefix+"0.**", "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00" + * |c1|t0|g1| tsfile + * |c2|t1|g1| dataset(dest) databasePrefix+"1.**" + * |c3|t1|g1| dataset(dest2) + * |c4|t2,t3|g2| dataset(dest) databasePrefix+"2.**", "now", null; databasePrefix+"3.**", null, "now" + * |c5|t3,t4|g2| dataset(dest2) databasePrefix+"4.**", null, "2024-03-31T23:59:59+08:00" + * |c6|t2,t4|g2| dataset(dest) + * |c7|t0,t3|g3| dataset(dest)/tsfile + * |c8|t6|g3| tsfile databasePrefix+"6.**", "now", null, + * |c9|t0,t3|g3| dataset(dest2)/tsfile + */ + @Test + public void do_test() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_0") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount00.addAndGet(1); + RowRecord next = dataset.next(); + // System.out.println("c0,g1:[rowCount00]" + + // next.getTimestamp() + "," + next.getFields()); + } + // + // System.out.println("c0,g1,rowCount00="+rowCount00.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount10.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(databasePrefix+0+".d_0:[rowCount10]" + + // next.getTimestamp() + "," + next.getFields()); + } + // + // System.out.println("c1,g1,rowCount10="+rowCount00.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_2") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_3") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_4") + .consumerGroupId("push_group_id_2") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_5") + .consumerGroupId("push_group_id_2") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_6") + .consumerGroupId("push_group_id_2") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_7") + .consumerGroupId("push_group_id_3") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount70.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(databasePrefix+"0.d_0:[rowCount70]" + + // next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_8") + .consumerGroupId("push_group_id_3") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "6.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount6.addAndGet(1); + RowRecord next = dataset.next(); + // System.out.println(databasePrefix+6+".d_0:" + + // next.getTimestamp() + "," + next.getFields()); + } + // + // System.out.println("c8,g3,rowCount00="+rowCount6.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_9") + .consumerGroupId("push_group_id_3") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount90.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(databasePrefix+"0.d_0:[rowCount90]" + + // next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + + for (int j = 0; j < consumers.size(); j++) { + consumers.get(j).open(); + } + + consumers.get(0).subscribe(topicNamePrefix + 0); + consumers.get(1).subscribe(topicNamePrefix + 0); + consumers.get(2).subscribe(topicNamePrefix + 1); + consumers.get(3).subscribe(topicNamePrefix + 1); + consumers.get(4).subscribe(topicNamePrefix + 2, topicNamePrefix + 3); + consumers.get(5).subscribe(topicNamePrefix + 3, topicNamePrefix + 4); + consumers.get(6).subscribe(topicNamePrefix + 2, topicNamePrefix + 4); + consumers.get(7).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + consumers.get(8).subscribe(topicNamePrefix + 6); + consumers.get(9).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + subs.getSubscriptions().forEach((System.out::println)); + + // Write data + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + for (int k = 0; k < 10; k++) { + String device = databasePrefix + k + ".d_0"; + try { + insert_data(timestamp, device, 20); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + } + timestamp += 40000; + } + for (int i = 0; i < 10; i++) { + String device = databasePrefix + i + ".d_0"; + try { + insert_data(System.currentTimeMillis(), device, 5); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + } + String device = databasePrefix + 2 + ".d_0"; + for (int i = 0; i < 20; i++) { + try { + insert_data(System.currentTimeMillis(), device, 5); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + } + }); + + thread.start(); + thread.join(); + + System.out.println(databasePrefix + "1.d_0:[src]" + getCount(session_src, sql1)); + System.out.println(databasePrefix + "2.d_0:[src]" + getCount(session_src, sql2)); + System.out.println(databasePrefix + "3.d_0:[src]" + getCount(session_src, sql3)); + System.out.println(databasePrefix + "4.d_0:[src]" + getCount(session_src, sql4)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount00.get() + rowCount10.get(), 400, "c0,c1,topic0,tsfile"); + assertEquals( + getCount(session_dest, sql1) + getCount(session_dest2, sql1), + getCount(session_src, sql1), + "c2,c3,topic1,group1"); + assertEquals( + getCount(session_dest, sql2) + getCount(session_dest2, sql2), + getCount(session_src, sql2) - 400, + "c4,c6,topic2,group2"); + final long topic3Total = getCount(session_dest, sql3) + getCount(session_dest2, sql3); + assertTrue(400 <= topic3Total && topic3Total <= 800, "c4,c5|c7,c9|topic3"); + assertEquals( + getCount(session_dest, sql4) + getCount(session_dest2, sql4), + 400, + "c5,c6,topic4,group3"); + assertGte(rowCount70.get() + rowCount90.get(), 400, "c7,c9,topic0,tsfile"); + assertEquals(rowCount6.get(), 5, "c8,topic6,tsfile"); + // assertTrue(rowCount00.get()>0); + // assertTrue(rowCount10.get()>0); + // assertTrue(rowCount70.get()>0); + // assertTrue(rowCount90.get()>0); + }); + } +} +/*** + * Expected result: + * root.test.MultiGroupVsMultiConsumer_1.d_0:[src]405 + * root.test.MultiGroupVsMultiConsumer_1.d_0:[dest]305 + * root.test.MultiGroupVsMultiConsumer_1.d_0:[dest2]100 + * root.test.MultiGroupVsMultiConsumer_2.d_0:[src]505 + * root.test.MultiGroupVsMultiConsumer_2.d_0:[dest]105 + * root.test.MultiGroupVsMultiConsumer_2.d_0:[dest2]0 + * root.test.MultiGroupVsMultiConsumer_3.d_0:[src]405 + * root.test.MultiGroupVsMultiConsumer_3.d_0:[dest]220 + * root.test.MultiGroupVsMultiConsumer_3.d_0:[dest2]300 + * root.test.MultiGroupVsMultiConsumer_4.d_0:[src]405 + * root.test.MultiGroupVsMultiConsumer_4.d_0:[dest]300 + * root.test.MultiGroupVsMultiConsumer_4.d_0:[dest2]100 + * rowCount00.get()=200 + * rowCount10.get()=200 + * rowCount70.get()=240 + * rowCount90.get()=160 + * rowCount6.get()=5 + **/ diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java new file mode 100644 index 0000000000000..e35e7f510a5d4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: fixed time range + * dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBOneConsumerMultiTopicsDatasetIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsDataset"; + private static final String database2 = "root.OneConsumerMultiTopicsDataset"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OneConsumerMultiTopicsDataset_1"; + private static List schemaList = new ArrayList<>(); + + private String pattern = database + ".**"; + private String pattern2 = database2 + ".**"; + private static final String device2 = database2 + ".d_1"; + private String topicName2 = "topic_OneConsumerMultiTopicsDataset_2"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-01T00:00:00+08:00", false); + createTopic_s(topicName2, pattern2, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,6.78);"); // 2024-03-13 08:00:00+08:00 + // Subscribe + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_multi") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + + consumer.subscribe(topicName, topicName2); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "show subscriptions after subscription"); + + Thread thread = + new Thread( + () -> { + try { + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + thread.start(); + thread.join(); + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-01T00:00:00+08:00"; + String sql2 = "select count(s_0) from " + device2; + System.out.println(FORMAT.format(new Date()) + " src device:" + getCount(session_src, sql)); + System.out.println(FORMAT.format(new Date()) + " src device:" + getCount(session_src, sql2)); + + AWAIT.untilAsserted( + () -> { + // Consumption data + check_count(5, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(10, "select count(s_0) from " + device2, "Consumption data:" + pattern2); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java new file mode 100644 index 0000000000000..1c7ee93c83ed3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: historical data + * The timing of flush is very critical. If the data inside the filter and the data outside the filter are within one tsfile, they will all be extracted. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBOneConsumerMultiTopicsMixIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsMix"; + private static final String database2 = "root.OneConsumerMultiTopicsMix"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OneConsumerMultiTopicsMix_1"; + private String pattern = database + ".**"; + private String pattern2 = database2 + ".**"; + private String device2 = database2 + ".d_0"; + private String topicName2 = "topic_OneConsumerMultiTopicsMix_2"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + createTopic_s(topicName2, pattern2, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,133.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + // Subscribe + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_mix_consumer_2_topic") + .consumerGroupId("OneConsumerMultiTopicsMix") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount1.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ":" + next.getTimestamp() + "," + next.getFields()); + } + dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device2, "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount2.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device2 + ":" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName, topicName2); + + System.out.println("###### Subscription Query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "subscribe and show subscriptions"); + // Subscribe and then write data + Thread thread = + new Thread( + () -> { + try { + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + thread.start(); + thread.join(); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount1.get(), 0, "pattern1"); + check_count(12, "select count(s_0) from " + device, "dataset pattern1"); + assertEquals(rowCount2.get(), 10, "pattern2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java new file mode 100644 index 0000000000000..f3be9db507d42 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: historical data + * The timing of flush is very critical. If the data inside the filter and the data outside the filter are within one tsfile, they will all be extracted. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBOneConsumerMultiTopicsTsfileIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsTsfile"; + private static final String database2 = "root.OneConsumerMultiTopicsTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OneConsumerMultiTopicsTsfile_1"; + private String pattern = database + ".**"; + private String pattern2 = database2 + ".**"; + private String device2 = database2 + ".d_0"; + private String topicName2 = "topic_OneConsumerMultiTopicsTsfile_2"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", true); + createTopic_s(topicName2, pattern2, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,133.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + // Subscribe + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile_consumer_2_topic") + .consumerGroupId("OneConsumerMultiTopicsTsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_0", true)), null)); + while (dataset.hasNext()) { + rowCount1.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ":" + next.getTimestamp() + "," + next.getFields()); + } + dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device2, "s_0", true)), null)); + while (dataset.hasNext()) { + rowCount2.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device2 + ":" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName, topicName2); + + System.out.println("###### Subscription query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "subscribe then show subscriptions"); + // Subscribe and then write data + Thread thread = + new Thread( + () -> { + try { + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + thread.start(); + thread.join(); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount1.get(), 7, "pattern1"); + assertEquals(rowCount2.get(), 5, "pattern2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDBPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDBPatternDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..59e93506fe4e2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDBPatternDatasetPushConsumerIT.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDBPatternDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBPatternDatasetPushConsumer"; + private static final String database2 = "root.DBPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DBPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consumption Data: s_0"); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption Data:s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDBPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDBPatternTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..96e6a93873b8a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDBPatternTsfilePushConsumerIT.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDBPatternTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DBPatternTsfilePushConsumer"; + private static final String database2 = "root.DBPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DBPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path(database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + dataset.next(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + assertTrue(onReceiveCount.get() >= 2, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 8, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 8, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 1, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + AWAIT.untilAsserted( + () -> { + assertTrue(onReceiveCount.get() >= 5, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 20, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 20, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 2, database + ".d_1.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDefaultPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDefaultPatternTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..d2fa1d2fdcd28 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDefaultPatternTsfilePushConsumerIT.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: root.** + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDefaultPatternTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DefaultPatternTsfilePushConsumer"; + private static final String database2 = "root.DefaultPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DefaultPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, null, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("default_pattern_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path(database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + dataset.next(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 8, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 8, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 1, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 1, database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + // Unsubscribe, and it will consume all again. + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 20, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 20, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 2, database + "d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 2, database2 + ".d_2.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDevicePatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDevicePatternDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..1cb49af0dccf0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDevicePatternDatasetPushConsumerIT.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: device + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDevicePatternDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DevicePatternDatasetPushConsumer"; + private static final String database2 = "root.DevicePatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DevicePatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".d_0.**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(12, "select count(s_1) from " + device, "Consumption data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDevicePatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDevicePatternTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..2e4b816c48a5e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBDevicePatternTsfilePushConsumerIT.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: device + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBDevicePatternTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.DevicePatternTsfilePushConsumer"; + private static final String database2 = "root.DevicePatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_DevicePatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = device + ".**"; + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = "select count(s_0) from " + device; + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path(database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + FORMAT.format(new Date()) + + ", " + + i + + ", " + + next.getTimestamp() + + "," + + next.getFields()); + } + } + System.out.println("onReceiveCount=" + onReceiveCount.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 10, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 10, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 25, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 25, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBRootPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBRootPatternDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..0ab26ee496759 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBRootPatternDatasetPushConsumerIT.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBRootPatternDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.RootPatternDatasetPushConsumer"; + private static final String database2 = "root.RootPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_RootPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = "root.**"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(10, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(10, "select count(s_1) from " + device, "Consumption Data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(1, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not preserved when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(15, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(15, "select count(s_1) from " + device, "Consumption Data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBTSPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBTSPatternDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..f4462e9a50ca3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBTSPatternDatasetPushConsumerIT.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTSPatternDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TSPatternDatasetPushConsumer"; + private static final String database2 = "root.TSPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TSPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".d_0.s_0"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBTSPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBTSPatternTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..7d0953c15927c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/pattern/IoTDBTSPatternTsfilePushConsumerIT.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTSPatternTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.TSPatternTsfilePushConsumer"; + private static final String database2 = "root.TSPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TSPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + private static SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + System.out.println("onReceiveCount=" + onReceiveCount.get()); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path(database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + dataset.next(); + } + System.out.println("rowCounts_" + i + ":" + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + String sql = "select count(s_0) from " + device; + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceiveCount.get(), 1, "receive files over 1"); + assertEquals(rowCounts.get(0).get(), 10, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + AWAIT.untilAsserted( + () -> { + assertGte(onReceiveCount.get(), 2, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 25, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBHistoryRootDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBHistoryRootDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..23cf1c0270f13 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBHistoryRootDatasetPushConsumerIT.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * AFTER + * pattern: root + * dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBHistoryRootDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.HistoryRootDatasetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = "root.**"; + private String topicName = "topic_HistoryRootDatasetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + Thread.sleep(5000); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_history_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(5, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(5, "select count(s_1) from " + device, "Consumption Data: s_1"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + AWAIT.untilAsserted( + () -> { + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + check_count(10, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(10, "select count(s_1) from " + device, "Consumption data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBHistoryRootTsFilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBHistoryRootTsFilePushConsumerIT.java new file mode 100644 index 0000000000000..ac661e6deac3f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBHistoryRootTsFilePushConsumerIT.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * Tsfile + * end_time: now + * pattern: root + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBHistoryRootTsFilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.HistoryRootTsFilePushConsumer"; + private String device = database + ".d_0"; + private String pattern = "root.**"; + private String topicName = "topic_HistoryRootTsFilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "2024-02-13T07:59:59+08:00", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerGroupId("push_time") + .consumerId("root_history_tsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount.get(), 4, "4 records"); + }); + + System.out.println("insert 2024-02-13 08:00:00+08:00"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount.get(), 4, "4 records"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBRealTimeDBDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBRealTimeDBDatasetPushConsumerIT.java new file mode 100644 index 0000000000000..08c8bbee31683 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBRealTimeDBDatasetPushConsumerIT.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBRealTimeDBDatasetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.RealTimeDBDatasetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_RealTimeDBDatasetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, "now", null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_realtime_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + check_count(0, "select count(s_0) from " + device, "Consumption data:" + pattern); + }); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + check_count(4, "select count(s_0) from " + device, "Consumption data:" + pattern); + }); + + // Subscribe and then write data + insert_data(System.currentTimeMillis() + 200000); // now + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consume data again:" + pattern); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBRealTimeDBTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBRealTimeDBTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..97452236b6b8f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBRealTimeDBTsfilePushConsumerIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBRealTimeDBTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.RealTimeDBTsfilePushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_RealTimeDBTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, "2024-01-31T08:02:00+08:00", null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_realtime_tsfile") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 0); + assertEquals(rowCount.get(), 0); + }); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1, "should process 1 file"); + assertGte(rowCount.get(), 4, "4 records"); + }); + + // Subscribe and then write data + insert_data(System.currentTimeMillis() + 200000); // now + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 2, "should process 2 file"); + assertGte(rowCount.get(), 8, "8 records"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeAccurateDBDataSetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeAccurateDBDataSetPushConsumerIT.java new file mode 100644 index 0000000000000..ac3b64fc8ffcd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeAccurateDBDataSetPushConsumerIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeRangeAccurateDBDataSetPushConsumerIT + extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeRangeAccurateDBDataSetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_TimeRangeAccurateDBDataSetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_time_accurate_range_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + // Before consumption subscription data + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + }); + + insert_data(System.currentTimeMillis()); // now + AWAIT.untilAsserted( + () -> { + check_count( + 2, + "select count(s_0) from " + device, + "After writing some real-time data:" + pattern); + }); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + check_count( + 6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + }); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + check_count(10, "select count(s_0) from " + device, "End time limit data:" + pattern); + }); + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + check_count(11, "select count(s_0) from " + device, "End time limit data 2:" + pattern); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeDBDataSetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeDBDataSetPushConsumerIT.java new file mode 100644 index 0000000000000..10a2ddb8c9209 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeDBDataSetPushConsumerIT.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * Start time, end time are both closed intervals. If not specified, the time will be 00:00:00. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeRangeDBDataSetPushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeRangeDBDataSetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_TimeRangeDBDataSetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T00:00:00+08:00", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_time_range_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + }); + + insert_data(System.currentTimeMillis()); // now + AWAIT.untilAsserted( + () -> { + check_count( + 2, "select count(s_0) from " + device, "Write some real-time data after:" + pattern); + }); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + check_count( + 6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + }); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + // Because the end time is 2024-03-31 00:00:00, closed interval + check_count(8, "select count(s_0) from " + device, "End time limit data:" + pattern); + }); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "End time limit data:" + pattern); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeDBTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeDBTsfilePushConsumerIT.java new file mode 100644 index 0000000000000..7ac75f60e16f1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/pushconsumer/time/IoTDBTimeRangeDBTsfilePushConsumerIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionConsumer; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * Start time, end time are both closed intervals. If not specified, the time will be 00:00:00. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionConsumer.class}) +public class IoTDBTimeRangeDBTsfilePushConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TimeRangeDBTsfilePushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_TimeRangeDBTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T00:00:00+08:00", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + consumer = + new SubscriptionTreePushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_time_range_accurate_tsfile") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1); + assertGte(rowCount.get(), 2); + }); + + insert_data(System.currentTimeMillis()); // now, not in range + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 1); + assertGte(rowCount.get(), 2); + }); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 2); + assertGte(rowCount.get(), 6); + }); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 3); + assertGte(rowCount.get(), 8); + }); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + assertGte(onReceive.get(), 3); + assertGte(rowCount.get(), 8); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/topic/IoTDBDataSet1TopicConsumerSpecialIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/topic/IoTDBDataSet1TopicConsumerSpecialIT.java new file mode 100644 index 0000000000000..2747e6dbc233d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/topic/IoTDBDataSet1TopicConsumerSpecialIT.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.topic; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Sequence-level topic, with start, end, tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBDataSet1TopicConsumerSpecialIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.test.ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz1"; + private String device = database + ".`#01`"; + private String pattern = device + ".`ABH#01`"; + private String topicName = "topic_DataSet1TopicConsumerSpecial"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-01T00:00:00+08:00", false); + session_src.createTimeseries( + pattern, TSDataType.INT32, TSEncoding.GORILLA, CompressionType.SNAPPY); + session_src.createTimeseries( + device + ".`BJ-ABH#01`", TSDataType.BOOLEAN, TSEncoding.RLE, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("`ABH#01`", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("`BJ-ABH#01`", TSDataType.BOOLEAN)); + System.out.println("topics:" + subs.getTopics()); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("`ABH#01`", rowIndex, row * 20 + row); + tablet.addValue("`BJ-ABH#01`", rowIndex, true); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,`ABH#01`,`BJ-ABH#01`)values(1710288000000,313,false);"); // 2024-03-13 + // 08:00:00+08:00 + // Subscribe + consumer = create_pull_consumer("`Group-ABH#01`", "`ABH#01`", false, 0L); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscribing, show subscriptions"); + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // Consumption data + consume_data(consumer); + check_count(4, "select count(`ABH#01`) from " + device, "Consumption data:" + pattern); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,`ABH#01`,`BJ-ABH#01`)values(1703980800000,1231,false);"); // 2023-12-31 + // 08:00:00+08:00 + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + // Consumption data + consume_data(consumer); + check_count(8, "select count(`ABH#01`) from " + device, "consume data again:" + pattern); + // Unsubscribe + consumer.unsubscribe(topicName); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/topic/IoTDBTestTopicNameIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/topic/IoTDBTestTopicNameIT.java new file mode 100644 index 0000000000000..45cddf8ebfe8b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/topic/IoTDBTestTopicNameIT.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.topic; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Special topic name + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBTestTopicNameIT extends AbstractSubscriptionTreeRegressionIT { + private String database = "root.TestTopicName"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "`1-group.1-consumer.ts.topic`"; + private List schemaList = new ArrayList<>(); + private SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + Thread.sleep(1000); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("g1", "c1", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + Thread.sleep(1000); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(10, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscribe"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + Thread.sleep(1000); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not preserved when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(15, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/user/IoTDBOtherUserConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/user/IoTDBOtherUserConsumerIT.java new file mode 100644 index 0000000000000..470d6ab1b39bd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/treemodel/regression/user/IoTDBOtherUserConsumerIT.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.treemodel.regression.user; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTreeRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.triple.treemodel.regression.AbstractSubscriptionTreeRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Permission Test: Username currently only serves for connection, no permissions defined. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTreeRegressionMisc.class}) +public class IoTDBOtherUserConsumerIT extends AbstractSubscriptionTreeRegressionIT { + private static final String database = "root.test.OtherUserConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OtherUserConsumer"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = "root.**"; + private static final String userName = "other_user"; + private static final String passwd = "other_user"; + private static SubscriptionTreePullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZ4); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + session_src.executeNonQueryStatement("drop user " + userName); + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + Thread.sleep(1000); + } + + // @Test + public void testPrivilege() throws IoTDBConnectionException, StatementExecutionException { + session_src.executeNonQueryStatement("create user " + userName + " '" + passwd + "';"); + session_src.executeNonQueryStatement("grant read,write on root.** to user " + userName); + } + + // @Test + // TODO: Failed to fetch all endpoints, only the admin user can perform this operation... + public void testNormal() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + session_src.executeNonQueryStatement("create user " + userName + " '" + passwd + "';"); + session_src.executeNonQueryStatement("grant read,write on root.** to user " + userName); + consumer = + new SubscriptionTreePullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .username(userName) + .password(passwd) + .buildPullConsumer(); + consumer.open(); + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + Thread.sleep(1000); + subs.getSubscriptions().forEach(System.out::println); + // Consumption data + consume_data(consumer, session_dest); + check_count(4, "select count(s_0) from " + device, "Consumption Data: s_0"); + check_count(4, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java index f04f387fd381a..cbbaf163ab98c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.tools.it; -import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.cli.it.AbstractScriptIT; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -40,7 +41,7 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class ExportDataTestIT extends AbstractScript { +public class ExportDataTestIT extends AbstractScriptIT { private static String ip; private static String port; @@ -75,12 +76,12 @@ public void test() throws IOException { @Override protected void testOnWindows() throws IOException { - final String[] output = {"Export completely!"}; + final String[] output = {Constants.EXPORT_COMPLETELY}; ProcessBuilder builder = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "export-data.bat", + toolsPath + File.separator + "windows" + File.separator + "export-data.bat", "-h", ip, "-p", @@ -89,6 +90,8 @@ protected void testOnWindows() throws IOException { "root", "-pw", "root", + "-ft", + "csv", "-t", "target", "-q", @@ -101,12 +104,12 @@ protected void testOnWindows() throws IOException { prepareData(); - final String[] output1 = {"Export completely!"}; + final String[] output1 = {Constants.EXPORT_COMPLETELY}; ProcessBuilder builder1 = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "export-data.bat", + toolsPath + File.separator + "windows" + File.separator + "export-data.bat", "-h", ip, "-p", @@ -115,6 +118,8 @@ protected void testOnWindows() throws IOException { "root", "-pw", "root", + "-ft", + "csv", "-t", "target", "-q", @@ -127,12 +132,12 @@ protected void testOnWindows() throws IOException { prepareData(); - final String[] output2 = {"Export completely!"}; + final String[] output2 = {Constants.EXPORT_COMPLETELY}; ProcessBuilder builder2 = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "export-data.bat", + toolsPath + File.separator + "windows" + File.separator + "export-data.bat", "-h", ip, "-p", @@ -143,7 +148,7 @@ protected void testOnWindows() throws IOException { "root", "-t", "target", - "-type", + "-ft", "sql", "-q", "select * from root.test.t2 where time > 1 and time < 1000000000000", @@ -156,7 +161,7 @@ protected void testOnWindows() throws IOException { @Override protected void testOnUnix() throws IOException { - final String[] output = {"Export completely!"}; + final String[] output = {Constants.EXPORT_COMPLETELY}; // -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**" ProcessBuilder builder = new ProcessBuilder( @@ -170,6 +175,8 @@ protected void testOnUnix() throws IOException { "root", "-pw", "root", + "-ft", + "csv", "-t", "target", "-q", @@ -179,7 +186,7 @@ protected void testOnUnix() throws IOException { prepareData(); - final String[] output1 = {"Export completely!"}; + final String[] output1 = {Constants.EXPORT_COMPLETELY}; // -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**" ProcessBuilder builder1 = new ProcessBuilder( @@ -195,6 +202,8 @@ protected void testOnUnix() throws IOException { "root", "-t", "target", + "-ft", + "csv", "-q", "select * from root.**"); builder1.environment().put("CLASSPATH", libPath); @@ -202,7 +211,7 @@ protected void testOnUnix() throws IOException { prepareData(); - final String[] output2 = {"Export completely!"}; + final String[] output2 = {Constants.EXPORT_COMPLETELY}; // -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**" ProcessBuilder builder2 = new ProcessBuilder( @@ -218,7 +227,7 @@ protected void testOnUnix() throws IOException { "root", "-t", "target", - "-type", + "-ft", "sql", "-q", "select * from root.test.t2 where time > 1 and time < 1000000000000"); diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java index f8a4cdae171b4..39325a47e413d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.tools.it; -import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.cli.it.AbstractScriptIT; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -41,7 +42,7 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class ExportSchemaTestIT extends AbstractScript { +public class ExportSchemaTestIT extends AbstractScriptIT { private static String ip; private static String port; @@ -81,12 +82,18 @@ public void test() throws IOException { @Override protected void testOnWindows() throws IOException { prepareSchema(); - final String[] output = {"Export completely!"}; + final String[] output = {Constants.EXPORT_COMPLETELY}; ProcessBuilder builder = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "export-schema.bat", + toolsPath + + File.separator + + "windows" + + File.separator + + "schema" + + File.separator + + "export-schema.bat", "-h", ip, "-p", @@ -109,11 +116,11 @@ protected void testOnWindows() throws IOException { @Override protected void testOnUnix() throws IOException { prepareSchema(); - final String[] output = {"Export completely!"}; + final String[] output = {Constants.EXPORT_COMPLETELY}; ProcessBuilder builder = new ProcessBuilder( "bash", - toolsPath + File.separator + "export-schema.sh", + toolsPath + File.separator + "schema" + File.separator + "export-schema.sh", "-h", ip, "-p", diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java index 5994d4d554eea..7579ecf9ee59f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tools.it; -import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.cli.it.AbstractScriptIT; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -41,7 +41,7 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class ExportTsFileTestIT extends AbstractScript { +public class ExportTsFileTestIT extends AbstractScriptIT { private static String ip; private static String port; @@ -76,38 +76,37 @@ public void test() throws IOException { @Override protected void testOnWindows() throws IOException { - final String[] output = {"!!!Warning:Tablet is empty,no data can be exported."}; - ProcessBuilder builder = - new ProcessBuilder( - "cmd.exe", - "/c", - toolsPath + File.separator + "export-tsfile.bat", - "-h", - ip, - "-p", - port, - "-u", - "root", - "-pw", - "root", - "-t", - "target", - "-q", - "select * from root.test.t2 where time > 1 and time < 1000000000000", - "&", - "exit", - "%^errorlevel%"); - builder.environment().put("CLASSPATH", libPath); - testOutput(builder, output, 1); + // Test for empty export, temporary removal + // final String[] output = {"Export TsFile Count: 0"}; + // ProcessBuilder builder = + // new ProcessBuilder( + // "cmd.exe", + // "/c", + // toolsPath + File.separator + "windows" + File.separator + "export-tsfile.bat", + // "-h", + // ip, + // "-p", + // port, + // "-u", + // "root", + // "-pw", + // "root", + // "-path", + // "root.test.t2.**", + // "&", + // "exit", + // "%^errorlevel%"); + // builder.environment().put("CLASSPATH", libPath); + // testOutput(builder, output, 0); prepareData(); - final String[] output1 = {"Export completely!"}; + final String[] output1 = {"Export TsFile Count: "}; ProcessBuilder builder1 = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "export-tsfile.bat", + toolsPath + File.separator + "windows" + File.separator + "export-tsfile.bat", "-h", ip, "-p", @@ -116,10 +115,8 @@ protected void testOnWindows() throws IOException { "root", "-pw", "root", - "-t", - "target", - "-q", - "select * from root.test.t2 where time > 1 and time < 1000000000000", + "-path", + "root.test.t2.**", "&", "exit", "%^errorlevel%"); @@ -129,30 +126,29 @@ protected void testOnWindows() throws IOException { @Override protected void testOnUnix() throws IOException { - final String[] output = {"!!!Warning:Tablet is empty,no data can be exported."}; - // -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**" - ProcessBuilder builder = - new ProcessBuilder( - "bash", - toolsPath + File.separator + "export-tsfile.sh", - "-h", - ip, - "-p", - port, - "-u", - "root", - "-pw", - "root", - "-t", - "target", - "-q", - "select * from root.**"); - builder.environment().put("CLASSPATH", libPath); - testOutput(builder, output, 1); + // Test for empty export, temporary removal + // final String[] output = {"Export TsFile Count: 0"}; + // // -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**" + // ProcessBuilder builder = + // new ProcessBuilder( + // "bash", + // toolsPath + File.separator + "export-tsfile.sh", + // "-h", + // ip, + // "-p", + // port, + // "-u", + // "root", + // "-pw", + // "root", + // "-path", + // "root.**"); + // builder.environment().put("CLASSPATH", libPath); + // testOutput(builder, output, 0); prepareData(); - final String[] output1 = {"Export completely!"}; + final String[] output1 = {"Export TsFile Count: "}; // -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**" ProcessBuilder builder1 = new ProcessBuilder( @@ -166,10 +162,8 @@ protected void testOnUnix() throws IOException { "root", "-pw", "root", - "-t", - "target", - "-q", - "select * from root.**"); + "-path", + "root.**"); builder1.environment().put("CLASSPATH", libPath); testOutput(builder1, output1, 0); } @@ -189,6 +183,7 @@ public void prepareData() { values.add("bbbbb"); values.add("abbes"); session.insertRecord(deviceId, 1L, measurements, values); + session.executeNonQueryStatement("flush"); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new RuntimeException(e); } diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java index 391b726f6d732..9d232dee02f11 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tools.it; -import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.cli.it.AbstractScriptIT; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -36,7 +36,7 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class ImportDataTestIT extends AbstractScript { +public class ImportDataTestIT extends AbstractScriptIT { private static String ip; @@ -74,13 +74,13 @@ public void test() throws IOException { @Override protected void testOnWindows() throws IOException { final String[] output = { - "The file name must end with \"csv\" or \"txt\"!", + "Source file or directory ./csv/ does not exist", }; ProcessBuilder builder = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "import-data.bat", + toolsPath + File.separator + "windows" + File.separator + "import-data.bat", "-h", ip, "-p", @@ -89,19 +89,21 @@ protected void testOnWindows() throws IOException { "root", "-pw", "root", + "-ft", + "csv", "-s", - "./", + "./csv/", "&", "exit", "%^errorlevel%"); builder.environment().put("CLASSPATH", libPath); - testOutput(builder, output, 0); + testOutput(builder, output, 1); } @Override protected void testOnUnix() throws IOException { final String[] output = { - "The file name must end with \"csv\" or \"txt\"!", + "Source file or directory ./csv/ does not exist", }; ProcessBuilder builder = new ProcessBuilder( @@ -115,9 +117,11 @@ protected void testOnUnix() throws IOException { "root", "-pw", "root", + "-ft", + "csv", "-s", - "./"); + "./csv/"); builder.environment().put("CLASSPATH", libPath); - testOutput(builder, output, 0); + testOutput(builder, output, 1); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java index ea7e1ba07f13f..f9a3908f726e4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tools.it; -import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.cli.it.AbstractScriptIT; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -36,7 +36,7 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class ImportSchemaTestIT extends AbstractScript { +public class ImportSchemaTestIT extends AbstractScriptIT { private static String ip; @@ -77,13 +77,19 @@ public void test() throws IOException { @Override protected void testOnWindows() throws IOException { final String[] output = { - "The file name must end with \"csv\"!", + "Source file or directory ./sql/ does not exist", }; ProcessBuilder builder = new ProcessBuilder( "cmd.exe", "/c", - toolsPath + File.separator + "import-schema.bat", + toolsPath + + File.separator + + "windows" + + File.separator + + "schema" + + File.separator + + "import-schema.bat", "-h", ip, "-p", @@ -92,24 +98,26 @@ protected void testOnWindows() throws IOException { "root", "-pw", "root", + "-db", + "test", "-s", - "./", + "./sql/", "&", "exit", "%^errorlevel%"); builder.environment().put("IOTDB_HOME", homePath); - testOutput(builder, output, 0); + testOutput(builder, output, 1); } @Override protected void testOnUnix() throws IOException { final String[] output = { - "The file name must end with \"csv\"!", + "Source file or directory ./sql/ does not exist", }; ProcessBuilder builder = new ProcessBuilder( "bash", - toolsPath + File.separator + "import-schema.sh", + toolsPath + File.separator + "schema" + File.separator + "import-schema.sh", "-h", ip, "-p", @@ -118,9 +126,11 @@ protected void testOnUnix() throws IOException { "root", "-pw", "root", + "-db", + "test", "-s", - "./"); + "./sql/"); builder.environment().put("IOTDB_HOME", homePath); - testOutput(builder, output, 0); + testOutput(builder, output, 1); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java index 7b8c2cb74dbb7..5ff95dee1d148 100644 --- a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.util; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunnerWithParametersFactory; diff --git a/integration-test/src/test/java/org/apache/iotdb/util/MagicUtils.java b/integration-test/src/test/java/org/apache/iotdb/util/MagicUtils.java new file mode 100644 index 0000000000000..e6213a72b71be --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/util/MagicUtils.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; + +public class MagicUtils { + + private static Logger LOGGER = LoggerFactory.getLogger(MagicUtils.class); + + /** + * Ignore all exceptions during close() + * + * @param t target object + * @return object which will close without exception + */ + public static T makeItCloseQuietly(T t) { + InvocationHandler handler = + (proxy, method, args) -> { + try { + if (method.getName().equals("close")) { + try { + method.invoke(t, args); + } catch (Throwable e) { + LOGGER.warn("Exception happens during close(): ", e); + } + return null; + } else { + return method.invoke(t, args); + } + } catch (InvocationTargetException e) { + throw e.getTargetException(); + } + }; + return (T) + Proxy.newProxyInstance( + t.getClass().getClassLoader(), t.getClass().getInterfaces(), handler); + } +} diff --git a/integration-test/src/test/resources/ainode-example/config.yaml b/integration-test/src/test/resources/ainode-example/config.yaml new file mode 100644 index 0000000000000..996e572cf39a6 --- /dev/null +++ b/integration-test/src/test/resources/ainode-example/config.yaml @@ -0,0 +1,5 @@ +configs: + input_shape: [7, 3] + output_shape: [7, 3] + input_type: ["float32", "float32","float32"] + output_type: ["float32", "float32","float32"] diff --git a/integration-test/src/test/resources/ainode-example/model.pt b/integration-test/src/test/resources/ainode-example/model.pt new file mode 100644 index 0000000000000..67d4aec6999f1 Binary files /dev/null and b/integration-test/src/test/resources/ainode-example/model.pt differ diff --git a/integration-test/src/test/resources/logback-test.xml b/integration-test/src/test/resources/logback-test.xml index 95a4d1c5c9cb2..ba595ff975232 100644 --- a/integration-test/src/test/resources/logback-test.xml +++ b/integration-test/src/test/resources/logback-test.xml @@ -49,7 +49,7 @@ - + diff --git a/iotdb-api/external-api/pom.xml b/iotdb-api/external-api/pom.xml index 860e1453757cb..589f90b00b37c 100644 --- a/iotdb-api/external-api/pom.xml +++ b/iotdb-api/external-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT external-api IoTDB: API: External API diff --git a/iotdb-api/pipe-api/pom.xml b/iotdb-api/pipe-api/pom.xml index c0e4338b3ef1f..b9e6b69bf73e7 100644 --- a/iotdb-api/pipe-api/pom.xml +++ b/iotdb-api/pipe-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT pipe-api IoTDB: API: Pipe API diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/PipePlugin.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/PipePlugin.java index ec06f5de31eb7..276c5c8b30089 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/PipePlugin.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/PipePlugin.java @@ -19,4 +19,41 @@ package org.apache.iotdb.pipe.api; +/** + * {@link PipePlugin} + * + *

{@link PipePlugin} represents a customizable component that can serve as a data extraction + * plugin, data processing plugin, or data sending plugin within a pipeline framework. + * + *

Developers can implement different plugin functionalities according to specific requirements, + * such as collecting data from various sources, transforming the data, or forwarding the data to + * external systems. + * + *

Usage Model: + * + *

    + *
  • By default, a {@link PipePlugin} can operate under a tree model only, if no model + * annotation under {@link org.apache.iotdb.pipe.api.annotation} is specified. + *
  • To extend its applicability, a {@link PipePlugin} can be configured to support table model, + * or both tree and table models concurrently. + *
+ * + *

Lifecycle: + * + *

    + *
  • When the pipeline framework loads, the plugin's configuration is parsed and validated. + *
  • As part of the setup, methods can be provided to prepare connections or resources required + * by the plugin (e.g., reading external configurations, establishing data routes). + *
  • During data processing, the plugin performs its core functionality (extraction, + * transformation, or sending). + *
  • When the pipeline is stopped or destroyed, any allocated resources must be released + * accurately, and {@link #close()} will be invoked to ensure a clean shutdown. + *
+ * + *

Example: {@link org.apache.iotdb.CountPointProcessor} + * + *

Implementations of {@link PipePlugin} should follow best practices for resource management and + * gracefully handle exceptions, especially when running in long-lived or continuously operating + * environments. + */ public interface PipePlugin extends AutoCloseable {} diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/annotation/TableModel.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/annotation/TableModel.java new file mode 100644 index 0000000000000..8a20d3b651304 --- /dev/null +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/annotation/TableModel.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.api.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that a plugin can be used in table model environments. + * + *

When implementing a custom {@link org.apache.iotdb.pipe.api.PipePlugin} that needs to operate + * under table model settings, declare this annotation on the plugin class. Through the {@code + * CREATE PIPEPLUGIN} statement, a plugin annotated with {@link TableModel} is valid for both tree + * model connections and table model connections. + * + * @since 2.0.0 + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface TableModel {} diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/annotation/TreeModel.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/annotation/TreeModel.java new file mode 100644 index 0000000000000..bc0fca6c52e4b --- /dev/null +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/annotation/TreeModel.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.api.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that a plugin can be used in tree model environments. + * + *

When implementing a custom {@link org.apache.iotdb.pipe.api.PipePlugin} that needs to operate + * under tree model settings, declare this annotation on the plugin class. Through the {@code CREATE + * PIPEPLUGIN} statement, a plugin annotated with {@link TreeModel} is valid for both tree model + * connections and tree model connections. + * + * @since 2.0.0 + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface TreeModel {} diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeRuntimeEnvironment.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeRuntimeEnvironment.java index 455d293dccc3b..8395568791cfc 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeRuntimeEnvironment.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeRuntimeEnvironment.java @@ -24,4 +24,6 @@ public interface PipeRuntimeEnvironment { String getPipeName(); long getCreationTime(); + + int getRegionId(); } diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameterValidator.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameterValidator.java index 1fa0046ccdab3..99d547a23c6d2 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameterValidator.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameterValidator.java @@ -23,6 +23,10 @@ import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class PipeParameterValidator { @@ -36,6 +40,40 @@ public PipeParameters getParameters() { return parameters; } + /** + * Validates whether the attributes entered by the user contain at least one attribute from + * lhsAttributes or rhsAttributes (if required), but not both. + * + * @param lhsAttributes list of left-hand side synonym attributes + * @param rhsAttributes list of right-hand side synonym attributes + * @param isRequired specifies whether at least one attribute from lhsAttributes or rhsAttributes + * must be provided + * @throws PipeParameterNotValidException if both lhsAttributes and rhsAttributes are provided + * @throws PipeAttributeNotProvidedException if isRequired is true and neither lhsAttributes nor + * rhsAttributes are provided + * @return the instance of PipeParameterValidator for method chaining + */ + public PipeParameterValidator validateSynonymAttributes( + final List lhsAttributes, + final List rhsAttributes, + final boolean isRequired) { + final boolean lhsExistence = lhsAttributes.stream().anyMatch(parameters::hasAttribute); + final boolean rhsExistence = rhsAttributes.stream().anyMatch(parameters::hasAttribute); + if (lhsExistence && rhsExistence) { + throw new PipeParameterNotValidException( + String.format( + "Cannot specify both %s and %s at the same time", lhsAttributes, rhsAttributes)); + } + if (isRequired && !lhsExistence && !rhsExistence) { + throw new PipeAttributeNotProvidedException( + Stream.concat(lhsAttributes.stream(), rhsAttributes.stream()) + .collect( + Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)) + .toString()); + } + return this; + } + /** * Validates whether the attributes entered by the user contain an attribute whose key is * attributeKey. @@ -83,7 +121,7 @@ public PipeParameterValidator validateAttributeValueRange( * @throws PipeParameterNotValidException if the given argument is not valid */ public PipeParameterValidator validate( - final PipeParameterValidator.SingleObjectValidationRule validationRule, + final SingleObjectValidationRule validationRule, final String messageToThrow, final Object argument) throws PipeParameterNotValidException { @@ -107,7 +145,7 @@ public interface SingleObjectValidationRule { * @throws PipeParameterNotValidException if the given arguments are not valid */ public PipeParameterValidator validate( - final PipeParameterValidator.MultipleObjectsValidationRule validationRule, + final MultipleObjectsValidationRule validationRule, final String messageToThrow, final Object... arguments) throws PipeParameterNotValidException { diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java index 1646423b133f6..afee5f5abbab5 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.function.BiFunction; import java.util.stream.Collectors; /** @@ -71,6 +72,25 @@ public boolean hasAnyAttributes(final String... keys) { return false; } + public void addAttribute(final String key, final String values) { + attributes.put(KeyReducer.reduce(key), values); + } + + public void computeAttributeIfExists( + final BiFunction valueComputer, final String... keys) { + for (String key : keys) { + if (attributes.containsKey(key)) { + attributes.compute(key, valueComputer); + return; + } + key = KeyReducer.reduce(key); + if (attributes.containsKey(key)) { + attributes.compute(key, valueComputer); + return; + } + } + } + public String getString(final String key) { final String value = attributes.get(key); return value != null ? value : attributes.get(KeyReducer.reduce(key)); @@ -379,6 +399,7 @@ public static class ValueHider { static { KEYS.add("ssl.trust-store-pwd"); + KEYS.add("password"); } static String hide(final String key, final String value) { diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConnectionException.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConnectionException.java index dc9d5e32968d0..ded27cc433a31 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConnectionException.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConnectionException.java @@ -24,11 +24,11 @@ public class PipeConnectionException extends PipeException { public static final String CONNECTION_ERROR_FORMATTER = "Error occurred while connecting to receiver %s:%s, please check network connectivity or SSL configurations when enable SSL transmission"; - public PipeConnectionException(String message) { + public PipeConnectionException(final String message) { super(message); } - public PipeConnectionException(String message, Throwable cause) { + public PipeConnectionException(final String message, final Throwable cause) { super(message, cause); } } diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusRetryWithIncreasingIntervalException.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusRetryWithIncreasingIntervalException.java new file mode 100644 index 0000000000000..8bc68ef9158ca --- /dev/null +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusRetryWithIncreasingIntervalException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.api.exception; + +public class PipeConsensusRetryWithIncreasingIntervalException extends PipeException { + + public PipeConsensusRetryWithIncreasingIntervalException(String message) { + super(message); + } +} diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeException.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeException.java index 3291f87be0de9..7be6d6a5560ce 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeException.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeException.java @@ -23,17 +23,17 @@ public class PipeException extends RuntimeException { private final long timeStamp; - public PipeException(String message) { + public PipeException(final String message) { super(message); this.timeStamp = System.currentTimeMillis(); } - public PipeException(String message, long timeStamp) { + public PipeException(final String message, final long timeStamp) { super(message); this.timeStamp = timeStamp; } - public PipeException(String message, Throwable cause) { + public PipeException(final String message, final Throwable cause) { super(message, cause); this.timeStamp = System.currentTimeMillis(); } diff --git a/iotdb-api/pom.xml b/iotdb-api/pom.xml index f29b64894b6f2..7d5db98efdfd8 100644 --- a/iotdb-api/pom.xml +++ b/iotdb-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-api pom diff --git a/iotdb-api/trigger-api/pom.xml b/iotdb-api/trigger-api/pom.xml index 49b192aebe70c..271bbfba6a823 100644 --- a/iotdb-api/trigger-api/pom.xml +++ b/iotdb-api/trigger-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT trigger-api IoTDB: API: Trigger API diff --git a/iotdb-api/udf-api/pom.xml b/iotdb-api/udf-api/pom.xml index f6b5897c826b9..570a7fa8be1de 100644 --- a/iotdb-api/udf-api/pom.xml +++ b/iotdb-api/udf-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT udf-api IoTDB: API: UDF API diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/AggregateFunctionAnalysis.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/AggregateFunctionAnalysis.java new file mode 100644 index 0000000000000..2eb4ba76aac67 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/AggregateFunctionAnalysis.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.customizer.analysis; + +import org.apache.iotdb.udf.api.type.Type; + +public class AggregateFunctionAnalysis implements FunctionAnalysis { + private final Type outputDataType; + private final boolean removable; + + private AggregateFunctionAnalysis(Type outputDataType, boolean removable) { + this.outputDataType = outputDataType; + this.removable = removable; + } + + public Type getOutputDataType() { + return outputDataType; + } + + public boolean isRemovable() { + return removable; + } + + public static class Builder { + private Type outputDataType; + private boolean removable = false; + + public Builder outputDataType(Type outputDataType) { + this.outputDataType = outputDataType; + return this; + } + + public Builder removable(boolean removable) { + this.removable = removable; + return this; + } + + public AggregateFunctionAnalysis build() throws IllegalArgumentException { + if (outputDataType == null) { + throw new IllegalArgumentException("AggregateFunctionAnalysis outputDataType is not set."); + } + return new AggregateFunctionAnalysis(outputDataType, removable); + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/FunctionAnalysis.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/FunctionAnalysis.java new file mode 100644 index 0000000000000..fec672330427c --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/FunctionAnalysis.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.customizer.analysis; + +/** + * FunctionAnalysis is an interface for analysis configurations of UDFs. It stores runtime + * attributes of UDF. + */ +public interface FunctionAnalysis {} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/ScalarFunctionAnalysis.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/ScalarFunctionAnalysis.java new file mode 100644 index 0000000000000..dc633203ed2fa --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/ScalarFunctionAnalysis.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.customizer.analysis; + +import org.apache.iotdb.udf.api.type.Type; + +public class ScalarFunctionAnalysis implements FunctionAnalysis { + + private final Type outputDataType; + + private ScalarFunctionAnalysis(Type outputDataType) { + this.outputDataType = outputDataType; + } + + public Type getOutputDataType() { + return outputDataType; + } + + public static class Builder { + private Type outputDataType; + + public Builder outputDataType(Type outputDataType) { + this.outputDataType = outputDataType; + return this; + } + + public ScalarFunctionAnalysis build() throws IllegalArgumentException { + if (outputDataType == null) { + throw new IllegalArgumentException("ScalarFunctionAnalysis outputDataType is not set."); + } + return new ScalarFunctionAnalysis(outputDataType); + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/config/AggregateFunctionConfig.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/config/AggregateFunctionConfig.java new file mode 100644 index 0000000000000..8257d76609aa7 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/config/AggregateFunctionConfig.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.customizer.config; + +import org.apache.iotdb.udf.api.relational.AggregateFunction; +import org.apache.iotdb.udf.api.type.Type; + +public class AggregateFunctionConfig extends UDFConfigurations { + + private boolean isRemovable = false; + + /** + * Set the output data type of the scalar function. + * + * @param outputDataType the output data type of the scalar function + * @return this + */ + public AggregateFunctionConfig setOutputDataType(Type outputDataType) { + this.outputDataType = outputDataType; + return this; + } + + /** + * If aggregate function is removable, {@linkplain AggregateFunction#remove} should be + * implemented. + * + * @param removable whether the aggregate function is removable + */ + public void setRemovable(boolean removable) { + isRemovable = removable; + } + + public boolean isRemovable() { + return isRemovable; + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/config/ScalarFunctionConfig.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/config/ScalarFunctionConfig.java new file mode 100644 index 0000000000000..c237e099b0dd8 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/config/ScalarFunctionConfig.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.customizer.config; + +import org.apache.iotdb.udf.api.type.Type; + +public class ScalarFunctionConfig extends UDFConfigurations { + + /** + * Set the output data type of the scalar function. + * + * @param outputDataType the output data type of the scalar function + * @return this + */ + public ScalarFunctionConfig setOutputDataType(Type outputDataType) { + this.outputDataType = outputDataType; + return this; + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java new file mode 100644 index 0000000000000..4b63c5a4f2f65 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.customizer.parameter; + +import org.apache.iotdb.udf.api.type.Type; + +import java.util.List; +import java.util.Map; + +/** + * FunctionArguments is used to provide the information of the function parameters to the UDF + * implementation. It contains the data types of the child expressions, system attributes, etc. + */ +public class FunctionArguments { + private final List argumentTypes; + private final Map systemAttributes; + + public FunctionArguments( + List childExpressionDataTypes, Map systemAttributes) { + this.argumentTypes = childExpressionDataTypes; + this.systemAttributes = systemAttributes; + } + + /** + * Get the data types of the arguments. + * + * @return a list of data types of the arguments + */ + public List getArgumentTypes() { + return argumentTypes; + } + + /** + * Get the number of the arguments. + * + * @return the number of the arguments + */ + public int getArgumentsSize() { + return argumentTypes.size(); + } + + /** + * Get the data type of the input child expression at the specified index. + * + * @param index column index + * @return the data type of the input child expression at the specified index + */ + public Type getDataType(int index) { + return argumentTypes.get(index); + } + + /** + * Check if the system attribute exists. + * + * @param attributeKey the key of the system attribute + * @return true if the system attribute exists, false otherwise + */ + public boolean hasSystemAttribute(String attributeKey) { + return systemAttributes.containsKey(attributeKey); + } + + /** + * Get all the system attributes. + * + * @return a map of the system attributes + */ + public Map getSystemAttributes() { + return systemAttributes; + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/exception/UDFArgumentNotValidException.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/exception/UDFArgumentNotValidException.java new file mode 100644 index 0000000000000..defa0826e0f56 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/exception/UDFArgumentNotValidException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.exception; + +public class UDFArgumentNotValidException extends UDFException { + + public UDFArgumentNotValidException(String message) { + super(message); + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/AggregateFunction.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/AggregateFunction.java new file mode 100644 index 0000000000000..6c9d385ac2467 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/AggregateFunction.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational; + +import org.apache.iotdb.udf.api.State; +import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.utils.ResultValue; + +public interface AggregateFunction extends SQLFunction { + + /** + * In this method, the user need to do the following things: + * + *

    + *
  • Validate {@linkplain FunctionArguments}. Throw {@link UDFArgumentNotValidException} if + * any parameter is not valid. + *
  • Use {@linkplain FunctionArguments} to get input data types and infer output data type. + *
  • Construct and return a {@linkplain AggregateFunctionAnalysis} object. + *
+ * + * @param arguments arguments used to validate + * @throws UDFArgumentNotValidException if any parameter is not valid + * @return the analysis result of the scalar function + */ + AggregateFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException; + + /** + * This method is called after the AggregateFunction is instantiated and before the beginning of + * the transformation process. This method is mainly used to initialize the resources used in + * AggregateFunction. + * + * @param arguments used to parse the input arguments entered by the user + * @throws UDFException the user can throw errors if necessary + */ + default void beforeStart(FunctionArguments arguments) throws UDFException { + // do nothing + } + + /** Create and initialize state. You may bind some resource in this method. */ + State createState(); + + /** + * Update state with data columns. + * + * @param state state to be updated + * @param input original input data row + */ + void addInput(State state, Record input); + + /** + * Merge two state in execution engine. + * + * @param state current state + * @param rhs right-hand-side state to be merged + */ + void combineState(State state, State rhs); + + /** + * Calculate output value from final state + * + * @param state final state + * @param resultValue used to collect output data points + */ + void outputFinal(State state, ResultValue resultValue); + + /** + * Remove input data from state. This method is used to remove the data points that have been + * added to the state. Once it is implemented, {@linkplain + * AggregateFunctionAnalysis.Builder#removable(boolean)} should be set to true. + * + * @param state state to be updated + * @param input row to be removed + */ + default void remove(State state, Record input) { + throw new UnsupportedOperationException(); + } + + /** This method is mainly used to release the resources used in the SQLFunction. */ + default void beforeDestroy() { + // do nothing + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/EmptyTableFunctionHandle.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/EmptyTableFunctionHandle.java new file mode 100644 index 0000000000000..8cba5ba385799 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/EmptyTableFunctionHandle.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational; + +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; + +public class EmptyTableFunctionHandle implements TableFunctionHandle { + @Override + public byte[] serialize() { + return new byte[0]; + } + + @Override + public void deserialize(byte[] bytes) {} + + @Override + public boolean equals(Object o) { + return o != null && getClass() == o.getClass(); + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/SQLFunction.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/SQLFunction.java new file mode 100644 index 0000000000000..e6a3846bc2ac0 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/SQLFunction.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational; + +public interface SQLFunction {} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/ScalarFunction.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/ScalarFunction.java new file mode 100644 index 0000000000000..68d7793093d2a --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/ScalarFunction.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational; + +import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.access.Record; + +public interface ScalarFunction extends SQLFunction { + /** + * In this method, the user need to do the following things: + * + *
    + *
  • Validate {@linkplain FunctionArguments}. Throw {@link UDFArgumentNotValidException} if + * any parameter is not valid. + *
  • Use {@linkplain FunctionArguments} to get input data types and infer output data type. + *
  • Construct and return a {@linkplain ScalarFunctionAnalysis} object. + *
+ * + * @param arguments arguments used to validate + * @throws UDFArgumentNotValidException if any parameter is not valid + * @return the analysis result of the scalar function + */ + ScalarFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException; + + /** + * This method is called after the ScalarFunction is instantiated and before the beginning of the + * transformation process. This method is mainly used to initialize the resources used in + * ScalarFunction. + * + * @param arguments used to parse the input arguments entered by the user + * @throws UDFException the user can throw errors if necessary + */ + default void beforeStart(FunctionArguments arguments) throws UDFException { + // do nothing + } + + /** + * This method will be called to process the transformation. In a single UDF query, this method + * may be called multiple times. + * + * @param input original input data row + * @throws UDFException the user can throw errors if necessary + */ + Object evaluate(Record input) throws UDFException; + + /** This method is mainly used to release the resources used in the ScalarFunction. */ + default void beforeDestroy() { + // do nothing + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/TableFunction.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/TableFunction.java new file mode 100644 index 0000000000000..a79c4336708ca --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/TableFunction.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; + +import java.util.List; +import java.util.Map; + +/** + * The result relation type of the table function consists of: + * + *
    + *
  • 1. columns created by the table function, called the proper columns. + *
  • 2. passed columns from input tables: + *
      + *
    • - for tables with the "pass through columns" option, these are all columns of the + * table, + *
    • - for tables without the "pass through columns" option, these are the partitioning + * columns of the table, if any. + *
    + *
+ */ +public interface TableFunction extends SQLFunction { + + /** + * This method is used to define the specification of the arguments required by the table + * function. Each argument is described using a {@link ParameterSpecification} object, which + * encapsulates details such as the argument name, whether it is required, its default value (if + * any), etc. + * + *

The {@link ParameterSpecification} class is abstract and has two concrete implementations: + * + *

    + *
  • {@link TableParameterSpecification}: Used for table parameter specification.We only + * support at most one table parameter. + *
  • {@link ScalarParameterSpecification}: Used for scalar parameter specification. We support + * any number of scalar parameters. + *
+ * + * @return a list of {@link ParameterSpecification} objects describing the function's arguments. + * The list should include all parameters that the table function expects, along with their + * constraints and default values (if applicable). + */ + List getArgumentsSpecifications(); + + /** + * This method is responsible for analyzing the provided arguments and constructing a {@link + * TableFunctionAnalysis} object in runtime. During analysis, the method should: + * + *
    + *
  • Extract values from scalar arguments (instances of {@link ScalarArgument}) or extract + * table descriptions from table arguments (instances of {@link TableArgument}). + *
  • Construct a {@link TableFunctionAnalysis} object that contains: + *
      + *
    • A description of proper columns. + *
    • A map indicating which columns from the input table arguments are required for the + * function to execute. + *
    • A TableFunctionExecutionInfo which stores all information necessary to execute the + * table function. + *
    + *
+ * + * @param arguments a map of argument names to their corresponding {@link Argument} values. The + * keys should match the parameter names defined in {@link #getArgumentsSpecifications()}. + * @throws UDFException if any argument is invalid or if an error occurs during analysis + * @return a {@link TableFunctionAnalysis} object containing the analysis result + */ + TableFunctionAnalysis analyze(Map arguments) throws UDFException; + + TableFunctionHandle createTableFunctionHandle(); + + /** + * This method is used to obtain a {@link TableFunctionProcessorProvider} that will be responsible + * for creating processors to handle the transformation of input data into output table. The + * provider is initialized with the validated arguments. + * + * @param tableFunctionHandle the object containing the execution information, which is generated + * in the {@link TableFunction#analyze} process. + * @return a {@link TableFunctionProcessorProvider} for creating processors + */ + TableFunctionProcessorProvider getProcessorProvider(TableFunctionHandle tableFunctionHandle); +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java new file mode 100644 index 0000000000000..8c6e2a7f3578c --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.access; + +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.utils.Binary; + +import java.time.LocalDate; + +public interface Record { + /** + * Returns the int value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code TSDataType.INT32}. + * + * @param columnIndex index of the specified column + * @return the int value at the specified column in this row + */ + int getInt(int columnIndex); + + /** + * Returns the long value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code TSDataType.INT64} + * or {@code TSDataType.TIMESTAMP}. + * + * @param columnIndex index of the specified column + * @return the long value at the specified column in this row + */ + long getLong(int columnIndex); + + /** + * Returns the float value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code TSDataType.FLOAT}. + * + * @param columnIndex index of the specified column + * @return the float value at the specified column in this row + */ + float getFloat(int columnIndex); + + /** + * Returns the double value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code + * TSDataType.DOUBLE}. + * + * @param columnIndex index of the specified column + * @return the double value at the specified column in this row + */ + double getDouble(int columnIndex); + + /** + * Returns the boolean value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code + * TSDataType.BOOLEAN}. + * + * @param columnIndex index of the specified column + * @return the boolean value at the specified column in this row + */ + boolean getBoolean(int columnIndex); + + /** + * Returns the Binary value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code TSDataType.TEXT}, + * {@code TSDataType.STRING} or {@code TSDataType.BLOB}. + * + * @param columnIndex index of the specified column + * @return the Binary value at the specified column in this row + */ + Binary getBinary(int columnIndex); + + /** + * Returns the String value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code TSDataType.TEXT} + * or {@code TSDataType.STRING}. + * + * @param columnIndex index of the specified column + * @return the String value at the specified column in this row + */ + String getString(int columnIndex); + + /** + * Returns the String value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code TSDataType.DATE}. + * + * @param columnIndex index of the specified column + * @return the String value at the specified column in this row + */ + LocalDate getLocalDate(int columnIndex); + + Object getObject(int columnIndex); + + /** + * Returns the actual data type of the value at the specified column in this row. + * + * @param columnIndex index of the specified column + * @return the actual data type of the value at the specified column in this row + */ + Type getDataType(int columnIndex); + + /** + * Returns {@code true} if the value of the specified column is null. + * + * @param columnIndex index of the specified column + * @return {@code true} if the value of the specified column is null + */ + boolean isNull(int columnIndex); + + /** + * Returns the number of columns. + * + * @return the number of columns + */ + int size(); +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/MapTableFunctionHandle.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/MapTableFunctionHandle.java new file mode 100644 index 0000000000000..da27eb22cd234 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/MapTableFunctionHandle.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table; + +import org.apache.iotdb.udf.api.type.Type; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class MapTableFunctionHandle implements TableFunctionHandle { + private static final Set> SUPPORT_VALUE_TYPE = + new HashSet<>( + Arrays.asList( + Integer.class, Long.class, Double.class, Float.class, String.class, Boolean.class)); + private final Map map = new HashMap<>(); + + public void addProperty(String key, Object value) { + if (!SUPPORT_VALUE_TYPE.contains(value.getClass())) { + throw new IllegalArgumentException("Unsupported value type."); + } + map.put(key, value); + } + + public Object getProperty(String key) { + return map.get(key); + } + + @Override + public byte[] serialize() { + ByteBuffer buffer = ByteBuffer.allocate(calculateSerializeSize()); + buffer.putInt(map.size()); + for (Map.Entry entry : map.entrySet()) { + byte[] bytes = entry.getKey().getBytes(StandardCharsets.UTF_8); + buffer.putInt(bytes.length); + buffer.put(bytes); + if (entry.getValue() instanceof Long) { + buffer.put(Type.INT64.getType()); + buffer.putLong((Long) entry.getValue()); + } else if (entry.getValue() instanceof Integer) { + buffer.put(Type.INT32.getType()); + buffer.putInt((Integer) entry.getValue()); + } else if (entry.getValue() instanceof Double) { + buffer.put(Type.DOUBLE.getType()); + buffer.putDouble((Double) entry.getValue()); + } else if (entry.getValue() instanceof Float) { + buffer.put(Type.FLOAT.getType()); + buffer.putFloat((Float) entry.getValue()); + } else if (entry.getValue() instanceof Boolean) { + buffer.put(Type.BOOLEAN.getType()); + buffer.put(Boolean.TRUE.equals(entry.getValue()) ? (byte) 1 : (byte) 0); + } else if (entry.getValue() instanceof String) { + buffer.put(Type.STRING.getType()); + bytes = ((String) entry.getValue()).getBytes(StandardCharsets.UTF_8); + buffer.putInt(bytes.length); + buffer.put(bytes); + } + } + return buffer.array(); + } + + private int calculateSerializeSize() { + int size = Integer.SIZE; + for (Map.Entry entry : map.entrySet()) { + size += Integer.BYTES + entry.getKey().getBytes(StandardCharsets.UTF_8).length + Byte.BYTES; + if (entry.getValue() instanceof Long) { + size += Long.BYTES; + } else if (entry.getValue() instanceof Integer) { + size += Integer.BYTES; + } else if (entry.getValue() instanceof Double) { + size += Double.BYTES; + } else if (entry.getValue() instanceof Float) { + size += Float.BYTES; + } else if (entry.getValue() instanceof Boolean) { + size += Byte.BYTES; + } else if (entry.getValue() instanceof String) { + byte[] bytes = ((String) entry.getValue()).getBytes(StandardCharsets.UTF_8); + size += Integer.BYTES + bytes.length; + } + } + return size; + } + + @Override + public void deserialize(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + int size = buffer.getInt(); + for (int i = 0; i < size; i++) { + byte[] b = new byte[buffer.getInt()]; + buffer.get(b); + String key = new String(b, StandardCharsets.UTF_8); + Type type = Type.valueOf(buffer.get()); + switch (type) { + case BOOLEAN: + map.put(key, buffer.get() != 0); + break; + case INT32: + map.put(key, buffer.getInt()); + break; + case INT64: + map.put(key, buffer.getLong()); + break; + case FLOAT: + map.put(key, buffer.getFloat()); + break; + case DOUBLE: + map.put(key, buffer.getDouble()); + break; + case STRING: + b = new byte[buffer.getInt()]; + buffer.get(b); + map.put(key, new String(b, StandardCharsets.UTF_8)); + break; + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("MapTableFunctionHandle{"); + for (Map.Entry entry : map.entrySet()) { + sb.append(entry.getKey()).append("=").append(entry.getValue()).append(", "); + } + if (sb.length() > 2) { + sb.setLength(sb.length() - 2); // remove last comma and space + } + sb.append('}'); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MapTableFunctionHandle handle = (MapTableFunctionHandle) o; + return Objects.equals(map, handle.map); + } + + @Override + public int hashCode() { + return Objects.hashCode(map); + } + + public static class Builder { + private final Map map = new HashMap<>(); + + public Builder addProperty(String key, Object value) { + if (!SUPPORT_VALUE_TYPE.contains(value.getClass())) { + throw new IllegalArgumentException("Unsupported value type."); + } + map.put(key, value); + return this; + } + + public MapTableFunctionHandle build() { + MapTableFunctionHandle handle = new MapTableFunctionHandle(); + handle.map.putAll(map); + return handle; + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionAnalysis.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionAnalysis.java new file mode 100644 index 0000000000000..630fa814fca0a --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionAnalysis.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table; + +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +/** + * An object of this class is produced by the `analyze()` method of a `TableFunction` + * implementation. It contains all the analysis results: + */ +public class TableFunctionAnalysis { + + /** + * The `properColumnSchema` field is used to inform the Analyzer of the proper columns returned by + * the Table Function, that is, the columns produced by the function. + */ + private final Optional properColumnSchema; + + /** + * The `requiredColumns` field is used to inform the Analyzer of the columns from the table + * arguments that are necessary to execute the table function. + */ + // a map from table argument name to list of column indexes for all columns required from the + // table argument + private final Map> requiredColumns; + + /** + * The `requireRecordSnapshot` field is used to inform the Analyzer of whether the table function + * processor requires a record snapshot. Default value is true, which means that the table + * function processor will receive a new record object for each record. Record object reference + * can be kept for future use. Otherwise, if it is false, the table function processor will always + * receive the same record object but with different values. It will perform better, but the + * processor must not keep the record object reference for future use. + */ + private final boolean requireRecordSnapshot; + + private final TableFunctionHandle handle; + + private TableFunctionAnalysis( + Optional properColumnSchema, + Map> requiredColumns, + boolean requiredRecordSnapshot, + TableFunctionHandle handle) { + this.properColumnSchema = requireNonNull(properColumnSchema, "returnedType is null"); + this.requiredColumns = requiredColumns; + this.requireRecordSnapshot = requiredRecordSnapshot; + this.handle = requireNonNull(handle, "TableFunctionHandle is null"); + } + + public Optional getProperColumnSchema() { + return properColumnSchema; + } + + public Map> getRequiredColumns() { + return requiredColumns; + } + + public boolean isRequireRecordSnapshot() { + return requireRecordSnapshot; + } + + public TableFunctionHandle getTableFunctionHandle() { + return handle; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private DescribedSchema properColumnSchema; + private final Map> requiredColumns = new HashMap<>(); + private boolean requireRecordSnapshot = true; + private TableFunctionHandle executionInfo; + + private Builder() {} + + public Builder properColumnSchema(DescribedSchema properColumnSchema) { + this.properColumnSchema = properColumnSchema; + return this; + } + + public Builder requiredColumns(String tableArgument, List columns) { + this.requiredColumns.put(tableArgument, columns); + return this; + } + + public Builder requireRecordSnapshot(boolean requireRecordSnapshot) { + this.requireRecordSnapshot = requireRecordSnapshot; + return this; + } + + public Builder handle(TableFunctionHandle executionInfo) { + this.executionInfo = executionInfo; + return this; + } + + public TableFunctionAnalysis build() { + return new TableFunctionAnalysis( + Optional.ofNullable(properColumnSchema), + requiredColumns, + requireRecordSnapshot, + executionInfo); + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionHandle.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionHandle.java new file mode 100644 index 0000000000000..86f85688f2a7b --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionHandle.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table; + +/** + * An area to store all information necessary to execute the table function, gathered at analysis + * time + */ +public interface TableFunctionHandle { + /** + * Serialize your state into byte array. The order of serialization must be consistent with + * deserialization. + */ + byte[] serialize(); + + /** + * Deserialize byte array into your state. The order of deserialization must be consistent with + * serialization. + */ + void deserialize(byte[] bytes); +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java new file mode 100644 index 0000000000000..fcbc1a198c1a4 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table; + +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionLeafProcessor; + +public interface TableFunctionProcessorProvider { + /** + * This method returns a {@linkplain TableFunctionDataProcessor}. All the necessary information + * collected during analysis is available in the implementation of TableFunctionProcessorProvider. + * It is called once per each partition processed by the table function. + */ + default TableFunctionDataProcessor getDataProcessor() { + throw new UnsupportedOperationException("this table function does not process input data"); + } + + /** + * This method returns a {@linkplain TableFunctionLeafProcessor}. All the necessary information + * collected during analysis is available in the implementation of TableFunctionProcessorProvider. + * It is called once per each split processed by the table function. + */ + default TableFunctionLeafProcessor getSplitProcessor() { + throw new UnsupportedOperationException("this table function does not process leaf data"); + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/Argument.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/Argument.java new file mode 100644 index 0000000000000..dcf68347082a1 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/Argument.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.argument; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public interface Argument { + + void serialize(ByteBuffer buffer); + + void serialize(DataOutputStream buffer) throws IOException; + + static Argument deserialize(ByteBuffer buffer) { + ArgumentType type = ArgumentType.values()[buffer.getInt()]; + switch (type) { + case TABLE_ARGUMENT: + return TableArgument.deserialize(buffer); + case SCALAR_ARGUMENT: + return ScalarArgument.deserialize(buffer); + default: + throw new IllegalArgumentException("Unknown argument type: " + type); + } + } + + enum ArgumentType { + TABLE_ARGUMENT, + SCALAR_ARGUMENT + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/DescribedSchema.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/DescribedSchema.java new file mode 100644 index 0000000000000..aecea22067f52 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/DescribedSchema.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.argument; + +import org.apache.iotdb.udf.api.type.Type; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class DescribedSchema { + private final List fields; + + private DescribedSchema(List fields) { + requireNonNull(fields, "fields is null"); + if (fields.isEmpty()) { + throw new IllegalArgumentException("DescribedSchema has no fields"); + } + this.fields = fields; + } + + public List getFields() { + return fields; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final List fields = new ArrayList<>(); + + public Builder addField(String name, Type type) { + fields.add(new Field(name, type)); + return this; + } + + public Builder addField(Optional name, Type type) { + fields.add(new Field(name, type)); + return this; + } + + public DescribedSchema build() { + return new DescribedSchema(fields); + } + } + + public static class Field { + private final Optional name; + private final Type type; + + public Field(String name, Type type) { + this.name = Optional.ofNullable(name); + this.type = type; + } + + public Field(Optional name, Type type) { + this.name = name; + this.type = type; + } + + public Optional getName() { + return name; + } + + public Type getType() { + return type; + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/ScalarArgument.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/ScalarArgument.java new file mode 100644 index 0000000000000..6ebe17bad8619 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/ScalarArgument.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.argument; + +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.utils.Binary; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.LocalDate; + +public class ScalarArgument implements Argument { + private final Type type; + private final Object value; + + public ScalarArgument(Type type, Object value) { + this.type = type; + this.value = value; + } + + public Type getType() { + return type; + } + + public Object getValue() { + return value; + } + + @Override + public void serialize(ByteBuffer buffer) { + buffer.putInt(ArgumentType.SCALAR_ARGUMENT.ordinal()); + buffer.put(type.getType()); + switch (type) { + case BOOLEAN: + buffer.put((boolean) value ? (byte) 1 : (byte) 0); + break; + case INT32: + buffer.putInt((int) value); + break; + case INT64: + case TIMESTAMP: + buffer.putLong((long) value); + break; + case FLOAT: + buffer.putFloat((float) value); + break; + case DOUBLE: + buffer.putDouble((double) value); + break; + case DATE: + buffer.putLong(((LocalDate) value).toEpochDay()); + break; + case TEXT: + case STRING: + byte[] bytes = ((String) value).getBytes(); + buffer.putInt(bytes.length); + buffer.put(bytes); + break; + case BLOB: + bytes = ((Binary) value).getValues(); + buffer.putInt(bytes.length); + buffer.put(bytes); + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } + + @Override + public void serialize(DataOutputStream buffer) throws IOException { + buffer.writeInt(ArgumentType.SCALAR_ARGUMENT.ordinal()); + buffer.writeByte(type.getType()); + switch (type) { + case BOOLEAN: + buffer.writeByte((byte) value); + break; + case INT32: + buffer.writeInt((int) value); + break; + case INT64: + case TIMESTAMP: + buffer.writeLong((long) value); + break; + case FLOAT: + buffer.writeFloat((float) value); + break; + case DOUBLE: + buffer.writeDouble((double) value); + break; + case DATE: + buffer.writeLong(((LocalDate) value).toEpochDay()); + break; + case TEXT: + case STRING: + byte[] bytes = ((String) value).getBytes(); + buffer.writeInt(bytes.length); + buffer.write(bytes); + break; + case BLOB: + bytes = ((Binary) value).getValues(); + buffer.writeInt(bytes.length); + buffer.write(bytes); + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } + + public static ScalarArgument deserialize(ByteBuffer buffer) { + Type type = Type.valueOf(buffer.get()); + switch (type) { + case BOOLEAN: + return new ScalarArgument(type, buffer.get() != 0); + case INT32: + return new ScalarArgument(type, buffer.getInt()); + case INT64: + case TIMESTAMP: + return new ScalarArgument(type, buffer.getLong()); + case FLOAT: + return new ScalarArgument(type, buffer.getFloat()); + case DOUBLE: + return new ScalarArgument(type, buffer.getDouble()); + case DATE: + return new ScalarArgument(type, LocalDate.ofEpochDay(buffer.getLong())); + case BLOB: + byte[] bytes = new byte[buffer.getInt()]; + buffer.get(bytes); + return new ScalarArgument(type, new Binary(bytes)); + case TEXT: + case STRING: + bytes = new byte[buffer.getInt()]; + buffer.get(bytes); + return new ScalarArgument(type, new String(bytes)); + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/ScalarArgumentChecker.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/ScalarArgumentChecker.java new file mode 100644 index 0000000000000..241ce5939bc88 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/ScalarArgumentChecker.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.argument; + +import java.util.function.Function; + +public class ScalarArgumentChecker { + public static Function POSITIVE_LONG_CHECKER = + (value) -> { + if (value instanceof Long && (Long) value > 0) { + return null; + } + return "should be a positive value"; + }; +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/TableArgument.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/TableArgument.java new file mode 100644 index 0000000000000..5b3b44be40042 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/argument/TableArgument.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.argument; + +import org.apache.iotdb.udf.api.type.Type; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class TableArgument implements Argument { + private final List> fieldNames; + private final List fieldTypes; + private final List partitionBy; + private final List orderBy; + private final boolean rowSemantics; + + public TableArgument( + List> fieldNames, + List fieldTypes, + List partitionBy, + List orderBy, + boolean rowSemantic) { + this.fieldNames = requireNonNull(fieldNames, "fieldNames is null"); + this.fieldTypes = requireNonNull(fieldTypes, "fieldTypes is null"); + if (fieldNames.size() != fieldTypes.size()) { + throw new IllegalArgumentException("fieldNames and fieldTypes must have the same size"); + } + this.partitionBy = requireNonNull(partitionBy, "partitionBy is null"); + this.orderBy = requireNonNull(orderBy, "orderBy is null"); + this.rowSemantics = rowSemantic; + } + + public List> getFieldNames() { + return fieldNames; + } + + public List getFieldTypes() { + return fieldTypes; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public boolean isRowSemantics() { + return rowSemantics; + } + + public int size() { + return fieldTypes.size(); + } + + @Override + public void serialize(ByteBuffer buffer) { + buffer.putInt(ArgumentType.TABLE_ARGUMENT.ordinal()); + buffer.putInt(fieldNames.size()); + for (Optional fieldName : fieldNames) { + if (fieldName.isPresent()) { + buffer.put((byte) 1); + byte[] bytes = fieldName.get().getBytes(); + buffer.putInt(bytes.length); + buffer.put(bytes); + } else { + buffer.put((byte) 0); + } + } + for (Type fieldType : fieldTypes) { + buffer.put(fieldType.getType()); + } + buffer.putInt(partitionBy.size()); + for (String partition : partitionBy) { + byte[] bytes = partition.getBytes(); + buffer.putInt(bytes.length); + buffer.put(bytes); + } + buffer.putInt(orderBy.size()); + for (String order : orderBy) { + byte[] bytes = order.getBytes(); + buffer.putInt(bytes.length); + buffer.put(bytes); + } + buffer.put((byte) (rowSemantics ? 1 : 0)); + } + + @Override + public void serialize(DataOutputStream buffer) throws IOException { + buffer.writeInt(ArgumentType.TABLE_ARGUMENT.ordinal()); + buffer.writeInt(fieldNames.size()); + for (Optional fieldName : fieldNames) { + if (fieldName.isPresent()) { + buffer.writeByte((byte) 1); + byte[] bytes = fieldName.get().getBytes(); + buffer.writeInt(bytes.length); + buffer.write(bytes); + } else { + buffer.writeByte((byte) 0); + } + } + for (Type fieldType : fieldTypes) { + buffer.writeByte(fieldType.getType()); + } + buffer.writeInt(partitionBy.size()); + for (String partition : partitionBy) { + byte[] bytes = partition.getBytes(); + buffer.writeInt(bytes.length); + buffer.write(bytes); + } + buffer.writeInt(orderBy.size()); + for (String order : orderBy) { + byte[] bytes = order.getBytes(); + buffer.writeInt(bytes.length); + buffer.write(bytes); + } + buffer.writeByte((byte) (rowSemantics ? 1 : 0)); + } + + public static TableArgument deserialize(ByteBuffer buffer) { + int size = buffer.getInt(); + List> fieldNames = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + if (buffer.get() == 1) { + byte[] bytes = new byte[buffer.getInt()]; + buffer.get(bytes); + fieldNames.add(Optional.of(new String(bytes))); + } else { + fieldNames.add(Optional.empty()); + } + } + List fieldTypes = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + fieldTypes.add(Type.valueOf(buffer.get())); + } + size = buffer.getInt(); + List partitionBy = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + byte[] bytes = new byte[buffer.getInt()]; + buffer.get(bytes); + partitionBy.add(new String(bytes)); + } + size = buffer.getInt(); + List orderBy = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + byte[] bytes = new byte[buffer.getInt()]; + buffer.get(bytes); + orderBy.add(new String(bytes)); + } + boolean rowSemantics = buffer.get() == 1; + return new TableArgument(fieldNames, fieldTypes, partitionBy, orderBy, rowSemantics); + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/processor/TableFunctionDataProcessor.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/processor/TableFunctionDataProcessor.java new file mode 100644 index 0000000000000..aa78e50f4d81c --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/processor/TableFunctionDataProcessor.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.processor; + +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.List; + +/** Each instance of TableFunctionDataProcessor processes one partition of data. */ +public interface TableFunctionDataProcessor { + + default void beforeStart() { + // do nothing + } + + /** + * This method processes a portion of data. It is called multiple times until the partition is + * fully processed. + * + * @param input {@link Record} including a portion of one partition for each table function's + * input table. A Record consists of columns requested during analysis (see {@link + * TableFunctionAnalysis#getRequiredColumns}). + * @param properColumnBuilders A list of {@link ColumnBuilder} for each column in the output + * table. + * @param passThroughIndexBuilder A {@link ColumnBuilder} for pass through columns. Index is + * started from 0 of the whole partition. It will be null if table argument is not declared + * with PASS_THROUGH_COLUMNS. + */ + void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder); + + /** + * This method is called after all data is processed. It is used to finalize the output table and + * close resource. All remaining data should be written to the columnBuilders. + * + * @param properColumnBuilders A list of {@link ColumnBuilder} for each column in the output + * table. + * @param passThroughIndexBuilder A {@link ColumnBuilder} for pass through columns. Index is + * started from 0 of the whole partition. It will be null if table argument is not declared + * with PASS_THROUGH_COLUMNS. + */ + default void finish( + List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { + // do nothing + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/processor/TableFunctionLeafProcessor.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/processor/TableFunctionLeafProcessor.java new file mode 100644 index 0000000000000..41750fc3b538e --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/processor/TableFunctionLeafProcessor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.processor; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.List; + +public interface TableFunctionLeafProcessor { + + default void beforeStart() { + // do nothing + } + + /** + * This method processes a portion of data. It is called multiple times until the processor is + * fully processed. + * + * @param columnBuilders a list of {@link ColumnBuilder} for each column in the output table. + */ + void process(List columnBuilders); + + /** This method is called to determine if the processor has finished processing all data. */ + boolean isFinish(); +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/ParameterSpecification.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/ParameterSpecification.java new file mode 100644 index 0000000000000..20b831d3568f3 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/ParameterSpecification.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.specification; + +import java.util.Optional; + +/** + * Abstract class to capture the three supported argument types for a table function: - Table + * arguments - Descriptor arguments - SQL scalar arguments + * + *

Each argument is named, and either passed positionally or in a `arg_name => value` convention. + * + *

Default values are allowed for all arguments except Table arguments. + */ +public abstract class ParameterSpecification { + private final String name; + private final boolean required; + + // native representation + private final Optional defaultValue; + + ParameterSpecification(String name, boolean required, Optional defaultValue) { + this.name = name; + this.required = required; + this.defaultValue = defaultValue; + if (required && defaultValue.isPresent()) { + throw new IllegalArgumentException("non-null default value for a required argument"); + } + } + + public String getName() { + return name; + } + + public boolean isRequired() { + return required; + } + + public Optional getDefaultValue() { + return defaultValue; + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/ScalarParameterSpecification.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/ScalarParameterSpecification.java new file mode 100644 index 0000000000000..84cc932cee501 --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/ScalarParameterSpecification.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.specification; + +import org.apache.iotdb.udf.api.type.Type; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +public class ScalarParameterSpecification extends ParameterSpecification { + private final Type type; + private final List> checkers; + + private ScalarParameterSpecification( + String name, + Type type, + boolean required, + Object defaultValue, + List> checkers) { + super(name, required, Optional.ofNullable(defaultValue)); + this.type = type; + if (defaultValue != null && !type.checkObjectType(defaultValue)) { + throw new IllegalArgumentException( + String.format( + "default value %s does not match the declared type: %s", defaultValue, type)); + } + this.checkers = checkers; + } + + public Type getType() { + return type; + } + + public List> getCheckers() { + return checkers; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private String name; + private Type type; + private boolean required = true; + private Object defaultValue; + private final List> checkers = new ArrayList<>(); + + private Builder() {} + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder type(Type type) { + this.type = type; + return this; + } + + public Builder addChecker(Function checker) { + this.checkers.add(checker); + return this; + } + + public Builder defaultValue(Object defaultValue) { + this.required = false; + this.defaultValue = defaultValue; + return this; + } + + public ScalarParameterSpecification build() { + return new ScalarParameterSpecification(name, type, required, defaultValue, checkers); + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/TableParameterSpecification.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/TableParameterSpecification.java new file mode 100644 index 0000000000000..a39af87dae15f --- /dev/null +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/specification/TableParameterSpecification.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.udf.api.relational.table.specification; + +import java.util.Optional; + +/** + * TableParameterSpecification are classified by two characteristics: + * + *
    + *
  • 1. Input tables have either row semantics or set semantics, as follows: + *
      + *
    • Row semantics means that the the result of the PTF as decided on a row-by-row basis. + * As an extreme example, the DBMS could atomize the input table into individual rows, + * and send each single row to a different processor. + *
    • Set semantics means that the outcome of the function depends on how the data is + * partitioned. A partition shall not be split across processors, nor may a processor + * handle more than one partition. + *
    + *
  • 2. Whether the input table supports pass-through columns or not. Pass-through columns isa + * mechanism enabling the PTF to copy every column of an input row into columns of an output + * row. + *
+ */ +public class TableParameterSpecification extends ParameterSpecification { + // set semantics or row semantics (default is set semantics) + private final boolean rowSemantics; + private final boolean passThroughColumns; + + private TableParameterSpecification( + String name, boolean rowSemantics, boolean passThroughColumns) { + // table arguments are always required + super(name, true, Optional.empty()); + this.rowSemantics = rowSemantics; + this.passThroughColumns = passThroughColumns; + } + + public boolean isRowSemantics() { + return rowSemantics; + } + + public boolean isPassThroughColumns() { + return passThroughColumns; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private String name; + private boolean rowSemantics; + private boolean passThroughColumns; + + private Builder() {} + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder setSemantics() { + this.rowSemantics = false; + return this; + } + + public Builder rowSemantics() { + this.rowSemantics = true; + return this; + } + + public Builder passThroughColumns() { + this.passThroughColumns = true; + return this; + } + + public TableParameterSpecification build() { + return new TableParameterSpecification(name, rowSemantics, passThroughColumns); + } + } +} diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java index 29aa5a45bb74c..8002d7b7ce504 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java @@ -19,6 +19,10 @@ package org.apache.iotdb.udf.api.type; +import org.apache.tsfile.utils.Binary; + +import java.time.LocalDate; + /** A substitution class for TsDataType in UDF APIs. */ public enum Type { /* BOOLEAN */ @@ -60,4 +64,38 @@ public enum Type { public byte getType() { return dataType; } + + public static Type valueOf(byte type) { + for (Type t : Type.values()) { + if (t.dataType == type) { + return t; + } + } + throw new IllegalArgumentException("Unsupported type: " + type); + } + + public boolean checkObjectType(Object o) { + switch (this) { + case BOOLEAN: + return o instanceof Boolean; + case INT32: + return o instanceof Integer; + case INT64: + case TIMESTAMP: + return o instanceof Long; + case FLOAT: + return o instanceof Float; + case DOUBLE: + return o instanceof Double; + case DATE: + return o instanceof LocalDate; + case BLOB: + return o instanceof Binary; + case STRING: + case TEXT: + return o instanceof String; + default: + return false; + } + } } diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index cc62e9e8d8d54..4bc3aad0c679f 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-cli IoTDB: Client: CLI @@ -37,37 +37,37 @@ org.apache.iotdb iotdb-session - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-jdbc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-antlr - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb node-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-server - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb isession - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb service-rpc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.tsfile @@ -82,12 +82,26 @@ org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT + + + org.apache.iotdb + iotdb-thrift-commons + 2.0.4-SNAPSHOT + + + org.apache.iotdb + pipe-api + 2.0.4-SNAPSHOT org.slf4j slf4j-api + + ch.qos.logback + logback-classic + org.antlr antlr4-runtime diff --git a/iotdb-client/cli/src/assembly/cli.xml b/iotdb-client/cli/src/assembly/cli.xml index c4e823f967fd7..17c240d7e2d28 100644 --- a/iotdb-client/cli/src/assembly/cli.xml +++ b/iotdb-client/cli/src/assembly/cli.xml @@ -39,5 +39,15 @@ src/assembly/resources ${file.separator} + + ${project.basedir}/../../scripts/sbin + sbin + + *cli.* + **/*cli.* + **/*cli-table.* + + 0755 + diff --git a/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.bat b/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.bat deleted file mode 100755 index bd1f4c9170cb0..0000000000000 --- a/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.bat +++ /dev/null @@ -1,50 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM - -@echo off - -@REM You can put your env variable here -@REM set JAVA_HOME=%JAVA_HOME% - -title IoTDB Load - -if "%OS%" == "Windows_NT" setlocal - -pushd %~dp0.. -if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% -popd - -if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ImportTsFile -if NOT DEFINED JAVA_HOME goto :err - -@REM ----------------------------------------------------------------------------- -@REM JVM Opts we'll use in legacy run or installation -set JAVA_OPTS=-ea^ - -DIOTDB_HOME="%IOTDB_HOME%" - -REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable. -if EXIST %IOTDB_HOME%\lib (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*" - -set PARAMETERS=%* - -echo Starting... -"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %PARAMETERS% -set ret_code=%ERRORLEVEL% - -EXIT /B %ret_code% diff --git a/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh b/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh deleted file mode 100755 index b7ded896c029e..0000000000000 --- a/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo --------------------- -echo Start Loading TsFile -echo --------------------- - -source "$(dirname "$0")/../sbin/iotdb-common.sh" -#get_iotdb_include and checkAllVariables is in iotdb-common.sh -VARS=$(get_iotdb_include "$*") -checkAllVariables -export IOTDB_HOME="${IOTDB_HOME}" -eval set -- "$VARS" - -PARAMETERS=$@ - -IOTDB_CLI_CONF=${IOTDB_HOME}/conf - -MAIN_CLASS=org.apache.iotdb.tool.ImportTsFile - -CLASSPATH="" -for f in ${IOTDB_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done - -if [ -n "$JAVA_HOME" ]; then - for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do - if [ -x "$java" ]; then - JAVA="$java" - break - fi - done -else - JAVA=java -fi - -set -o noglob -iotdb_cli_params="-Dlogback.configurationFile=${IOTDB_CLI_CONF}/logback-tool.xml" - -echo "Starting..." -exec "$JAVA" $iotdb_cli_params -cp "$CLASSPATH" "$MAIN_CLASS" $PARAMETERS - -exit $? diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index 0da407e00f00f..50be4ea5d5ac4 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -20,13 +20,14 @@ package org.apache.iotdb.cli; import org.apache.iotdb.cli.utils.CliContext; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.iotdb.jdbc.IoTDBJDBCResultSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.service.rpc.thrift.ServerProperties; -import org.apache.iotdb.tool.ImportData; +import org.apache.iotdb.tool.data.ImportData; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; @@ -37,7 +38,6 @@ import org.apache.tsfile.utils.DateUtils; import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStreamReader; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -48,8 +48,11 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; +import static org.apache.iotdb.jdbc.Config.SQL_DIALECT; + public abstract class AbstractCli { static final String HOST_ARGS = "h"; @@ -97,7 +100,8 @@ public abstract class AbstractCli { static final String SET_FETCH_SIZE = "set fetch_size"; static final String SHOW_FETCH_SIZE = "show fetch_size"; private static final String HELP = "help"; - static final String IOTDB_CLI_PREFIX = "IoTDB"; + static final String IOTDB = "IoTDB"; + static String cliPrefix = IOTDB; static final String SCRIPT_HINT = "./start-cli.sh(start-cli.bat if Windows)"; static final String QUIT_COMMAND = "quit"; static final String EXIT_COMMAND = "exit"; @@ -140,6 +144,9 @@ public abstract class AbstractCli { static int lastProcessStatus = CODE_OK; + static String sqlDialect = "tree"; + static String usingDatabase = null; + static void init() { keywordSet.add("-" + HOST_ARGS); keywordSet.add("-" + HELP_ARGS); @@ -156,11 +163,12 @@ static void init() { static Options createOptions() { Options options = new Options(); - Option help = new Option(HELP_ARGS, false, "Display help information(optional)"); + Option help = new Option(HELP_ARGS, false, "Display help information. (optional)"); help.setRequired(false); options.addOption(help); - Option timeFormat = new Option(ISO8601_ARGS, false, "Display timestamp in number(optional)"); + Option timeFormat = + new Option(ISO8601_ARGS, false, "Display timestamp in numeric format. (optional)"); timeFormat.setRequired(false); options.addOption(timeFormat); @@ -168,7 +176,7 @@ static Options createOptions() { Option.builder(HOST_ARGS) .argName(HOST_NAME) .hasArg() - .desc("Host Name (optional, default 127.0.0.1)") + .desc("Host Name. Default is 127.0.0.1. (optional)") .build(); options.addOption(host); @@ -176,7 +184,7 @@ static Options createOptions() { Option.builder(PORT_ARGS) .argName(PORT_NAME) .hasArg() - .desc("Port (optional, default 6667)") + .desc("Port. Default is 6667. (optional)") .build(); options.addOption(port); @@ -184,20 +192,24 @@ static Options createOptions() { Option.builder(USERNAME_ARGS) .argName(USERNAME_NAME) .hasArg() - .desc("User name (required)") + .desc("User name. (required)") .required() .build(); options.addOption(username); Option password = - Option.builder(PW_ARGS).argName(PW_NAME).hasArg().desc("password (optional)").build(); + Option.builder(PW_ARGS) + .argName(PW_NAME) + .hasArg() + .desc("Password. Default is root. (optional)") + .build(); options.addOption(password); Option useSSL = Option.builder(USE_SSL_ARGS) .argName(USE_SSL) .hasArg() - .desc("use_ssl statement (optional)") + .desc("Use SSL statement. (optional)") .build(); options.addOption(useSSL); @@ -205,7 +217,7 @@ static Options createOptions() { Option.builder(TRUST_STORE_ARGS) .argName(TRUST_STORE) .hasArg() - .desc("trust_store statement (optional)") + .desc("Trust store statement. (optional)") .build(); options.addOption(trustStore); @@ -213,7 +225,7 @@ static Options createOptions() { Option.builder(TRUST_STORE_PWD_ARGS) .argName(TRUST_STORE_PWD) .hasArg() - .desc("trust_store_pwd statement (optional)") + .desc("Trust store password statement. (optional)") .build(); options.addOption(trustStorePwd); @@ -221,14 +233,14 @@ static Options createOptions() { Option.builder(EXECUTE_ARGS) .argName(EXECUTE_NAME) .hasArg() - .desc("execute statement (optional)") + .desc("Execute a statement. (optional)") .build(); options.addOption(execute); Option isRpcCompressed = Option.builder(RPC_COMPRESS_ARGS) .argName(RPC_COMPRESS_NAME) - .desc("Rpc Compression enabled or not") + .desc("Enable or disable Rpc Compression. (optional)") .build(); options.addOption(isRpcCompressed); @@ -236,11 +248,17 @@ static Options createOptions() { Option.builder(TIMEOUT_ARGS) .argName(TIMEOUT_NAME) .hasArg() - .desc( - "The timeout in second. " - + "Using the configuration of server if it's not set (optional)") + .desc("The timeout in seconds. Uses the server configuration if not set. (optional)") .build(); options.addOption(queryTimeout); + + Option sqlDialect = + Option.builder(SQL_DIALECT) + .argName(SQL_DIALECT) + .hasArg() + .desc("Currently supports tree and table; uses tree if not set. (optional)") + .build(); + options.addOption(sqlDialect); return options; } @@ -255,15 +273,12 @@ static String checkRequiredArg( String str = commandLine.getOptionValue(arg); if (str == null) { if (isRequired) { - String msg = - String.format( - "%s: Required values for option '%s' not provided", IOTDB_CLI_PREFIX, name); + String msg = String.format("%s: Required values for option '%s' not provided", IOTDB, name); ctx.getPrinter().println(msg); ctx.getPrinter().println("Use -help for more information"); throw new ArgsErrorException(msg); } else if (defaultValue == null) { - String msg = - String.format("%s: Required values for option '%s' is null.", IOTDB_CLI_PREFIX, name); + String msg = String.format("%s: Required values for option '%s' is null.", IOTDB, name); throw new ArgsErrorException(msg); } else { return defaultValue; @@ -308,6 +323,10 @@ static void setQueryTimeout(String timeoutString) { } } + static void setSqlDialect(String sqlDialect) { + AbstractCli.sqlDialect = sqlDialect; + } + static String[] removePasswordArgs(String[] args) { int index = -1; for (int i = 0; i < args.length; i++) { @@ -553,6 +572,7 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri ZoneId zoneId = ZoneId.of(connection.getTimeZone()); statement.setFetchSize(fetchSize); boolean hasResultSet = statement.execute(cmd.trim()); + long costTime = System.currentTimeMillis() - startTime; if (hasResultSet) { // print the result try (ResultSet resultSet = statement.getResultSet()) { @@ -562,7 +582,6 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri List> lists = cacheResult(ctx, resultSet, maxSizeList, columnLength, resultSetMetaData, zoneId); output(ctx, lists, maxSizeList); - long costTime = System.currentTimeMillis() - startTime; ctx.getPrinter().println(String.format("It costs %.3fs", costTime / 1000.0)); while (!isReachEnd) { if (continuePrint) { @@ -574,9 +593,16 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri } ctx.getPrinter() .println("This display 1000 rows. Press ENTER to show more, input 'q' to quit."); - BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); + + BufferedReader br = new BufferedReader(new InputStreamReader(ctx.getIn())); try { - if ("".equals(br.readLine())) { + String line; + if (ctx.isDisableCliHistory()) { + line = ctx.getLineReader().readLine(); + } else { + line = br.readLine(); + } + if ("".equals(line)) { maxSizeList = new ArrayList<>(columnLength); lists = cacheResult( @@ -585,7 +611,7 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri } else { break; } - } catch (IOException e) { + } catch (Exception e) { ctx.getPrinter().printException(e); executeStatus = CODE_ERROR; } @@ -604,6 +630,8 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri ctx.getPrinter().println("Msg: " + e); executeStatus = CODE_ERROR; } finally { + updateSqlDialectAndUsingDatabase( + connection.getParams().getSqlDialect(), connection.getParams().getDb().orElse(null)); resetArgs(); } return executeStatus; @@ -718,7 +746,7 @@ private static List> cacheResult( private static String getStringByColumnIndex( IoTDBJDBCResultSet resultSet, int columnIndex, ZoneId zoneId) throws SQLException { - TSDataType type = TSDataType.valueOf(resultSet.getColumnTypeByIndex(columnIndex)); + TSDataType type = resultSet.getColumnTypeByIndex(columnIndex); switch (type) { case BOOLEAN: case INT32: @@ -806,12 +834,14 @@ private static void output(CliContext ctx, List> lists, List ", null); + s = ctx.getLineReader().readLine(cliPrefix + "> ", null); boolean continues = processCommand(ctx, s, connection); if (!continues) { return true; @@ -223,6 +230,11 @@ private static boolean readerReadLine(CliContext ctx, IoTDBConnection connection } catch (EndOfFileException e) { // Exit on EOF (usually by pressing CTRL+D). ctx.exit(CODE_OK); + } catch (IllegalArgumentException e) { + if (e.getMessage().contains("history")) { + return false; + } + throw e; } return false; } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/CliContext.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/CliContext.java index 2085dea5a6821..1edd2d928e952 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/CliContext.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/CliContext.java @@ -36,14 +36,26 @@ public class CliContext { private final ExitType exitType; + private final boolean disableCliHistory; + private LineReader lineReader; public CliContext(InputStream in, PrintStream out, PrintStream err, ExitType exitType) { + this(in, out, err, exitType, false); + } + + public CliContext( + InputStream in, + PrintStream out, + PrintStream err, + ExitType exitType, + boolean disableCliHistory) { this.in = in; this.out = out; this.err = err; this.exitType = exitType; this.printer = new IoTPrinter(out); + this.disableCliHistory = disableCliHistory; } public InputStream getIn() { @@ -66,6 +78,10 @@ public ExitType getExitType() { return exitType; } + public boolean isDisableCliHistory() { + return disableCliHistory; + } + public LineReader getLineReader() { return lineReader; } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/JlineUtils.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/JlineUtils.java index d46d72bcdcd9d..c902598cd3f1c 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/JlineUtils.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/utils/JlineUtils.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.util.Objects; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -54,6 +56,8 @@ private JlineUtils() {} public static LineReader getLineReader(CliContext ctx, String username, String host, String port) throws IOException { + Logger.getLogger("org.jline").setLevel(Level.OFF); + // Defaulting to a dumb terminal when a supported terminal can not be correctly created // see https://github.com/jline/jline3/issues/291 Terminal terminal; @@ -77,21 +81,27 @@ public static LineReader getLineReader(CliContext ctx, String username, String h LineReaderBuilder builder = LineReaderBuilder.builder(); builder.terminal(terminal); - // Handle the command history. By default, the number of commands will not exceed 500 and the - // size of the history fill will be less than 10 KB. See: - // org.jline.reader.impl.history#DefaultHistory - String historyFile = ".iotdb_history"; - String historyFilePath = - System.getProperty("user.home") - + File.separator - + historyFile - + "-" - + host.hashCode() - + "-" - + port - + "-" - + username.hashCode(); - builder.variable(LineReader.HISTORY_FILE, new File(historyFilePath)); + if (!ctx.isDisableCliHistory()) { + // Handle the command history. By default, the number of commands will not exceed 500 and the + // size of the history fill will be less than 10 KB. See: + // org.jline.reader.impl.history#DefaultHistory + String historyFile = ".iotdb_history"; + String historyFilePath = + System.getProperty("user.home") + + File.separator + + historyFile + + "-" + + host.hashCode() + + "-" + + port + + "-" + + username.hashCode(); + builder.variable(LineReader.HISTORY_FILE, new File(historyFilePath)); + // Set the maximum number of history records in the history file to 1,000 lines. + builder.variable(LineReader.HISTORY_FILE_SIZE, 1_000); + // Set the maximum number of history records in memory to 1,000 lines. + builder.variable(LineReader.HISTORY_SIZE, 1_000); + } // TODO: since the lexer doesn't produce tokens for quotation marks, disable the highlighter to // avoid incorrect inputs. @@ -107,6 +117,18 @@ public static LineReader getLineReader(CliContext ctx, String username, String h org.jline.reader.impl.DefaultParser parser = new org.jline.reader.impl.DefaultParser(); builder.parser(parser); LineReader lineReader = builder.build(); + + try { + // Load the history file, if loading fails, purge the history file. + lineReader.getHistory().attach(lineReader); + lineReader.getHistory().load(); + } catch (Exception e) { + try { + lineReader.getHistory().purge(); + } catch (Exception ex) { + // Ignore + } + } if (OSUtils.IS_WINDOWS) { // If enabled cursor remains in begin parenthesis (gitbash). lineReader.setVariable(LineReader.BLINK_MATCHING_PAREN, 0); diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java deleted file mode 100644 index 1e689e903c1ab..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.QuoteMode; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.PrintWriter; -import java.time.ZoneId; -import java.util.List; - -public abstract class AbstractDataTool { - - protected static final String HOST_ARGS = "h"; - protected static final String HOST_NAME = "host"; - protected static final String HOST_DEFAULT_VALUE = "127.0.0.1"; - - protected static final String HELP_ARGS = "help"; - - protected static final String PORT_ARGS = "p"; - protected static final String PORT_NAME = "port"; - protected static final String PORT_DEFAULT_VALUE = "6667"; - - protected static final String PW_ARGS = "pw"; - protected static final String PW_NAME = "password"; - protected static final String PW_DEFAULT_VALUE = "root"; - - protected static final String USERNAME_ARGS = "u"; - protected static final String USERNAME_NAME = "username"; - protected static final String USERNAME_DEFAULT_VALUE = "root"; - - protected static final String TIME_FORMAT_ARGS = "tf"; - protected static final String TIME_FORMAT_NAME = "timeformat"; - - protected static final String TIME_ZONE_ARGS = "tz"; - protected static final String TIME_ZONE_NAME = "timeZone"; - - protected static final String TIMEOUT_ARGS = "timeout"; - protected static final String TIMEOUT_NAME = "timeout"; - protected static final int MAX_HELP_CONSOLE_WIDTH = 92; - protected static final String[] TIME_FORMAT = - new String[] {"default", "long", "number", "timestamp"}; - protected static final String[] STRING_TIME_FORMAT = - new String[] { - "yyyy-MM-dd HH:mm:ss.SSSX", - "yyyy/MM/dd HH:mm:ss.SSSX", - "yyyy.MM.dd HH:mm:ss.SSSX", - "yyyy-MM-dd HH:mm:ssX", - "yyyy/MM/dd HH:mm:ssX", - "yyyy.MM.dd HH:mm:ssX", - "yyyy-MM-dd HH:mm:ss.SSSz", - "yyyy/MM/dd HH:mm:ss.SSSz", - "yyyy.MM.dd HH:mm:ss.SSSz", - "yyyy-MM-dd HH:mm:ssz", - "yyyy/MM/dd HH:mm:ssz", - "yyyy.MM.dd HH:mm:ssz", - "yyyy-MM-dd HH:mm:ss.SSS", - "yyyy/MM/dd HH:mm:ss.SSS", - "yyyy.MM.dd HH:mm:ss.SSS", - "yyyy-MM-dd HH:mm:ss", - "yyyy/MM/dd HH:mm:ss", - "yyyy.MM.dd HH:mm:ss", - "yyyy-MM-dd'T'HH:mm:ss.SSSX", - "yyyy/MM/dd'T'HH:mm:ss.SSSX", - "yyyy.MM.dd'T'HH:mm:ss.SSSX", - "yyyy-MM-dd'T'HH:mm:ssX", - "yyyy/MM/dd'T'HH:mm:ssX", - "yyyy.MM.dd'T'HH:mm:ssX", - "yyyy-MM-dd'T'HH:mm:ss.SSSz", - "yyyy/MM/dd'T'HH:mm:ss.SSSz", - "yyyy.MM.dd'T'HH:mm:ss.SSSz", - "yyyy-MM-dd'T'HH:mm:ssz", - "yyyy/MM/dd'T'HH:mm:ssz", - "yyyy.MM.dd'T'HH:mm:ssz", - "yyyy-MM-dd'T'HH:mm:ss.SSS", - "yyyy/MM/dd'T'HH:mm:ss.SSS", - "yyyy.MM.dd'T'HH:mm:ss.SSS", - "yyyy-MM-dd'T'HH:mm:ss", - "yyyy/MM/dd'T'HH:mm:ss", - "yyyy.MM.dd'T'HH:mm:ss" - }; - protected static final int CODE_OK = 0; - protected static final int CODE_ERROR = 1; - - protected static String host; - protected static String port; - protected static String username; - protected static String password; - protected static ZoneId zoneId; - - protected static String timeZoneID; - protected static String timeFormat; - protected static String exportType; - protected static String aligned; - protected static Session session; - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDataTool.class); - - protected AbstractDataTool() {} - - protected static String checkRequiredArg( - String arg, String name, CommandLine commandLine, String defaultValue) - throws ArgsErrorException { - String str = commandLine.getOptionValue(arg); - if (str == null) { - if (StringUtils.isNotBlank(defaultValue)) { - return defaultValue; - } - String msg = String.format("Required values for option '%s' not provided", name); - LOGGER.info(msg); - LOGGER.info("Use -help for more information"); - throw new ArgsErrorException(msg); - } - return str; - } - - protected static void setTimeZone() throws IoTDBConnectionException, StatementExecutionException { - if (timeZoneID != null) { - session.setTimeZone(timeZoneID); - } - zoneId = ZoneId.of(session.getTimeZone()); - } - - protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException { - host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, HOST_DEFAULT_VALUE); - port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, PORT_DEFAULT_VALUE); - username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, USERNAME_DEFAULT_VALUE); - password = commandLine.getOptionValue(PW_ARGS, PW_DEFAULT_VALUE); - } - - protected static boolean checkTimeFormat() { - for (String format : TIME_FORMAT) { - if (timeFormat.equals(format)) { - return true; - } - } - for (String format : STRING_TIME_FORMAT) { - if (timeFormat.equals(format)) { - return true; - } - } - LOGGER.info( - "Input time format {} is not supported, " - + "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n", - timeFormat); - return false; - } - - protected static Options createNewOptions() { - Options options = new Options(); - - Option opHost = - Option.builder(HOST_ARGS) - .longOpt(HOST_NAME) - .argName(HOST_NAME) - .hasArg() - .desc("Host Name (optional)") - .build(); - options.addOption(opHost); - - Option opPort = - Option.builder(PORT_ARGS) - .longOpt(PORT_NAME) - .argName(PORT_NAME) - .hasArg() - .desc("Port (optional)") - .build(); - options.addOption(opPort); - - Option opUsername = - Option.builder(USERNAME_ARGS) - .longOpt(USERNAME_NAME) - .argName(USERNAME_NAME) - .hasArg() - .desc("Username (optional)") - .build(); - options.addOption(opUsername); - - Option opPassword = - Option.builder(PW_ARGS) - .longOpt(PW_NAME) - .optionalArg(true) - .argName(PW_NAME) - .hasArg() - .desc("Password (optional)") - .build(); - options.addOption(opPassword); - return options; - } - - /** - * write data to CSV file. - * - * @param headerNames the header names of CSV file - * @param records the records of CSV file - * @param filePath the directory to save the file - */ - public static Boolean writeCsvFile( - List headerNames, List> records, String filePath) { - try { - final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(filePath); - if (headerNames != null) { - csvPrinterWrapper.printRecord(headerNames); - } - for (List CsvRecord : records) { - csvPrinterWrapper.printRecord(CsvRecord); - } - csvPrinterWrapper.flush(); - csvPrinterWrapper.close(); - return true; - } catch (IOException e) { - ioTPrinter.printException(e); - return false; - } - } - - static class CSVPrinterWrapper { - private final String filePath; - private final CSVFormat csvFormat; - private CSVPrinter csvPrinter; - - public CSVPrinterWrapper(String filePath) { - this.filePath = filePath; - this.csvFormat = - CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setEscape('\\') - .setQuoteMode(QuoteMode.NONE) - .build(); - } - - public void printRecord(final Iterable values) throws IOException { - if (csvPrinter == null) { - csvPrinter = csvFormat.print(new PrintWriter(filePath)); - } - csvPrinter.printRecord(values); - } - - public void print(Object value) { - if (csvPrinter == null) { - try { - csvPrinter = csvFormat.print(new PrintWriter(filePath)); - } catch (IOException e) { - ioTPrinter.printException(e); - return; - } - } - try { - csvPrinter.print(value); - } catch (IOException e) { - ioTPrinter.printException(e); - } - } - - public void println() throws IOException { - csvPrinter.println(); - } - - public void close() throws IOException { - if (csvPrinter != null) { - csvPrinter.close(); - } - } - - public void flush() throws IOException { - if (csvPrinter != null) { - csvPrinter.flush(); - } - } - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java deleted file mode 100644 index 9e91d404d1211..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.QuoteMode; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - -public abstract class AbstractSchemaTool { - - protected static final String HOST_ARGS = "h"; - protected static final String HOST_NAME = "host"; - protected static final String HOST_DEFAULT_VALUE = "127.0.0.1"; - - protected static final String HELP_ARGS = "help"; - - protected static final String PORT_ARGS = "p"; - protected static final String PORT_NAME = "port"; - protected static final String PORT_DEFAULT_VALUE = "6667"; - - protected static final String PW_ARGS = "pw"; - protected static final String PW_NAME = "password"; - protected static final String PW_DEFAULT_VALUE = "root"; - - protected static final String USERNAME_ARGS = "u"; - protected static final String USERNAME_NAME = "username"; - protected static final String USERNAME_DEFAULT_VALUE = "root"; - - protected static final String TIMEOUT_ARGS = "timeout"; - protected static final String TIMEOUT_ARGS_NAME = "queryTimeout"; - - protected static final int MAX_HELP_CONSOLE_WIDTH = 92; - - protected static final int CODE_OK = 0; - protected static final int CODE_ERROR = 1; - - protected static String host; - protected static String port; - protected static String username; - protected static String password; - - protected static String aligned; - protected static Session session; - - protected static final List HEAD_COLUMNS = - Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression"); - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class); - - protected AbstractSchemaTool() {} - - protected static String checkRequiredArg( - String arg, String name, CommandLine commandLine, String defaultValue) - throws ArgsErrorException { - String str = commandLine.getOptionValue(arg); - if (str == null) { - if (StringUtils.isNotBlank(defaultValue)) { - return defaultValue; - } - String msg = String.format("Required values for option '%s' not provided", name); - LOGGER.info(msg); - LOGGER.info("Use -help for more information"); - throw new ArgsErrorException(msg); - } - return str; - } - - protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException { - host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, HOST_DEFAULT_VALUE); - port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, PORT_DEFAULT_VALUE); - username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, USERNAME_DEFAULT_VALUE); - password = checkRequiredArg(PW_ARGS, PW_NAME, commandLine, PW_DEFAULT_VALUE); - } - - protected static Options createNewOptions() { - Options options = new Options(); - - Option opHost = - Option.builder(HOST_ARGS) - .longOpt(HOST_NAME) - .optionalArg(true) - .argName(HOST_NAME) - .hasArg() - .desc("Host Name (optional)") - .build(); - options.addOption(opHost); - - Option opPort = - Option.builder(PORT_ARGS) - .longOpt(PORT_NAME) - .optionalArg(true) - .argName(PORT_NAME) - .hasArg() - .desc("Port (optional)") - .build(); - options.addOption(opPort); - - Option opUsername = - Option.builder(USERNAME_ARGS) - .longOpt(USERNAME_NAME) - .optionalArg(true) - .argName(USERNAME_NAME) - .hasArg() - .desc("Username (optional)") - .build(); - options.addOption(opUsername); - - Option opPassword = - Option.builder(PW_ARGS) - .longOpt(PW_NAME) - .optionalArg(true) - .argName(PW_NAME) - .hasArg() - .desc("Password (optional)") - .build(); - options.addOption(opPassword); - return options; - } - - /** - * write data to CSV file. - * - * @param records the records of CSV file - * @param filePath the directory to save the file - */ - public static Boolean writeCsvFile(List> records, String filePath) { - try { - final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(filePath); - for (List CsvRecord : records) { - csvPrinterWrapper.printRecordln(CsvRecord); - } - csvPrinterWrapper.flush(); - csvPrinterWrapper.close(); - return true; - } catch (IOException e) { - ioTPrinter.printException(e); - return false; - } - } - - static class CSVPrinterWrapper { - private final String filePath; - private final CSVFormat csvFormat; - private CSVPrinter csvPrinter; - - public CSVPrinterWrapper(String filePath) { - this.filePath = filePath; - this.csvFormat = - CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setEscape('\\') - .setQuoteMode(QuoteMode.NONE) - .build(); - } - - public void printRecord(final Iterable values) throws IOException { - if (csvPrinter == null) { - csvPrinter = csvFormat.print(new PrintWriter(filePath)); - } - csvPrinter.printRecord(values); - } - - public void printRecordln(final Iterable values) throws IOException { - if (csvPrinter == null) { - csvPrinter = csvFormat.print(new PrintWriter(filePath)); - } - Iterator var2 = values.iterator(); - - while (var2.hasNext()) { - Object value = var2.next(); - csvPrinter.print(value); - } - csvPrinter.println(); - } - - public void print(Object value) { - if (csvPrinter == null) { - try { - csvPrinter = csvFormat.print(new PrintWriter(filePath)); - } catch (IOException e) { - ioTPrinter.printException(e); - return; - } - } - try { - csvPrinter.print(value); - } catch (IOException e) { - ioTPrinter.printException(e); - } - } - - public void println() throws IOException { - csvPrinter.println(); - } - - public void close() throws IOException { - if (csvPrinter != null) { - csvPrinter.close(); - } - } - - public void flush() throws IOException { - if (csvPrinter != null) { - csvPrinter.flush(); - } - } - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java deleted file mode 100644 index 08918bcf2af28..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java +++ /dev/null @@ -1,589 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.type.ExitType; -import org.apache.iotdb.cli.utils.CliContext; -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.cli.utils.JlineUtils; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.RpcUtils; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.thrift.TException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.Field; -import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.common.RowRecord; -import org.jline.reader.LineReader; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; - -/** - * Export CSV file. - * - * @version 1.0.0 20170719 - */ -public class ExportData extends AbstractDataTool { - - private static final String TARGET_DIR_ARGS = "t"; - private static final String TARGET_DIR_NAME = "targetDirectory"; - - private static final String TARGET_FILE_ARGS = "tfn"; - private static final String TARGET_FILE_NAME = "targetFileName"; - - private static final String SQL_FILE_ARGS = "s"; - private static final String SQL_FILE_NAME = "sourceSqlFile"; - - private static final String DATA_TYPE_ARGS = "datatype"; - private static final String DATA_TYPE_NAME = "datatype"; - - private static final String QUERY_COMMAND_ARGS = "q"; - private static final String QUERY_COMMAND_NAME = "queryCommand"; - - private static final String EXPORT_TYPE_ARGS = "type"; - - private static final String EXPORT_TYPE_NAME = "exportType"; - - private static final String EXPORT_SQL_TYPE_NAME = "sql"; - - private static final String ALIGNED_ARGS = "aligned"; - private static final String ALIGNED_NAME = "export aligned insert sql"; - private static final String LINES_PER_FILE_ARGS = "lpf"; - private static final String LINES_PER_FILE_ARGS_NAME = "linesPerFile"; - - private static final String TSFILEDB_CLI_PREFIX = "ExportData"; - - private static final String DUMP_FILE_NAME_DEFAULT = "dump"; - private static String targetFile = DUMP_FILE_NAME_DEFAULT; - - private static String targetDirectory; - - private static Boolean needDataTypePrinted; - - private static String queryCommand; - - private static String timestampPrecision; - - private static int linesPerFile = 10000; - - private static long timeout = -1; - - private static Boolean aligned = false; - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - - @SuppressWarnings({ - "squid:S3776", - "squid:S2093" - }) // Suppress high Cognitive Complexity warning, ignore try-with-resources - /* main function of export csv tool. */ - public static void main(String[] args) { - Options options = createOptions(); - HelpFormatter hf = new HelpFormatter(); - CommandLine commandLine = null; - CommandLineParser parser = new DefaultParser(); - hf.setOptionComparator(null); // avoid reordering - hf.setWidth(MAX_HELP_CONSOLE_WIDTH); - - if (args == null || args.length == 0) { - ioTPrinter.println("Too few params input, please check the following hint."); - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - try { - commandLine = parser.parse(options, args); - } catch (ParseException e) { - ioTPrinter.println(e.getMessage()); - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - if (commandLine.hasOption(HELP_ARGS)) { - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - int exitCode = CODE_OK; - try { - parseBasicParams(commandLine); - parseSpecialParams(commandLine); - if (!checkTimeFormat()) { - System.exit(CODE_ERROR); - } - session = new Session(host, Integer.parseInt(port), username, password); - session.open(false); - timestampPrecision = session.getTimestampPrecision(); - setTimeZone(); - - if (queryCommand == null) { - String sqlFile = commandLine.getOptionValue(SQL_FILE_ARGS); - String sql; - - if (sqlFile == null) { - LineReader lineReader = - JlineUtils.getLineReader( - new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), - username, - host, - port); - sql = lineReader.readLine(TSFILEDB_CLI_PREFIX + "> please input query: "); - ioTPrinter.println(sql); - String[] values = sql.trim().split(";"); - for (int i = 0; i < values.length; i++) { - dumpResult(values[i], i); - } - } else { - dumpFromSqlFile(sqlFile); - } - } else { - dumpResult(queryCommand, 0); - } - - } catch (IOException e) { - ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (ArgsErrorException e) { - ioTPrinter.println("Invalid args: " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (IoTDBConnectionException | StatementExecutionException e) { - ioTPrinter.println("Connect failed because " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (TException e) { - ioTPrinter.println( - "Can not get the timestamp precision from server because " + e.getMessage()); - exitCode = CODE_ERROR; - } finally { - if (session != null) { - try { - session.close(); - } catch (IoTDBConnectionException e) { - exitCode = CODE_ERROR; - ioTPrinter.println( - "Encounter an error when closing session, error is: " + e.getMessage()); - } - } - } - System.exit(exitCode); - } - - private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { - targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_NAME, commandLine, null); - targetFile = commandLine.getOptionValue(TARGET_FILE_ARGS); - needDataTypePrinted = Boolean.valueOf(commandLine.getOptionValue(DATA_TYPE_ARGS)); - queryCommand = commandLine.getOptionValue(QUERY_COMMAND_ARGS); - exportType = commandLine.getOptionValue(EXPORT_TYPE_ARGS); - String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS); - if (timeoutString != null) { - timeout = Long.parseLong(timeoutString); - } - if (needDataTypePrinted == null) { - needDataTypePrinted = true; - } - if (targetFile == null) { - targetFile = DUMP_FILE_NAME_DEFAULT; - } - timeFormat = commandLine.getOptionValue(TIME_FORMAT_ARGS); - if (timeFormat == null) { - timeFormat = "default"; - } - timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS); - if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { - targetDirectory += File.separator; - } - if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) { - linesPerFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS)); - } - if (commandLine.getOptionValue(ALIGNED_ARGS) != null) { - aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS)); - } - } - - /** - * commandline option create. - * - * @return object Options - */ - private static Options createOptions() { - Options options = createNewOptions(); - - Option opTargetFile = - Option.builder(TARGET_DIR_ARGS) - .required() - .argName(TARGET_DIR_NAME) - .hasArg() - .desc("Target File Directory (required)") - .build(); - options.addOption(opTargetFile); - - Option targetFileName = - Option.builder(TARGET_FILE_ARGS) - .argName(TARGET_FILE_NAME) - .hasArg() - .desc("Export file name (optional)") - .build(); - options.addOption(targetFileName); - - Option opSqlFile = - Option.builder(SQL_FILE_ARGS) - .argName(SQL_FILE_NAME) - .hasArg() - .desc("SQL File Path (optional)") - .build(); - options.addOption(opSqlFile); - - Option opTimeFormat = - Option.builder(TIME_FORMAT_ARGS) - .argName(TIME_FORMAT_NAME) - .hasArg() - .desc( - "Output time Format in csv file. " - + "You can choose 1) timestamp, number, long 2) ISO8601, default 3) " - + "user-defined pattern like yyyy-MM-dd HH:mm:ss, default ISO8601.\n OutPut timestamp in sql file, No matter what time format is set(optional)") - .build(); - options.addOption(opTimeFormat); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); - - Option opDataType = - Option.builder(DATA_TYPE_ARGS) - .argName(DATA_TYPE_NAME) - .hasArg() - .desc( - "Will the data type of timeseries be printed in the head line of the CSV file?" - + '\n' - + "You can choose true) or false) . (optional)") - .build(); - options.addOption(opDataType); - - Option opQuery = - Option.builder(QUERY_COMMAND_ARGS) - .argName(QUERY_COMMAND_NAME) - .hasArg() - .desc("The query command that you want to execute. (optional)") - .build(); - options.addOption(opQuery); - - Option opTypeQuery = - Option.builder(EXPORT_TYPE_ARGS) - .argName(EXPORT_TYPE_NAME) - .hasArg() - .desc("Export file type ?" + '\n' + "You can choose csv) or sql) . (optional)") - .build(); - options.addOption(opTypeQuery); - - Option opAligned = - Option.builder(ALIGNED_ARGS) - .argName(ALIGNED_NAME) - .hasArg() - .desc("Whether export to sql of aligned (only sql optional)") - .build(); - options.addOption(opAligned); - - Option opLinesPerFile = - Option.builder(LINES_PER_FILE_ARGS) - .argName(LINES_PER_FILE_ARGS_NAME) - .hasArg() - .desc("Lines per dump file.") - .build(); - options.addOption(opLinesPerFile); - - Option opHelp = - Option.builder(HELP_ARGS) - .longOpt(HELP_ARGS) - .hasArg(false) - .desc("Display help information") - .build(); - options.addOption(opHelp); - - Option opTimeout = - Option.builder(TIMEOUT_ARGS) - .longOpt(TIMEOUT_NAME) - .hasArg() - .desc("Timeout for session query") - .build(); - options.addOption(opTimeout); - return options; - } - - /** - * This method will be called, if the query commands are written in a sql file. - * - * @param filePath sql file path - * @throws IOException exception - */ - private static void dumpFromSqlFile(String filePath) throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - String sql; - int index = 0; - while ((sql = reader.readLine()) != null) { - dumpResult(sql, index); - index++; - } - } - } - - /** - * Dump files from database to CSV file. - * - * @param sql export the result of executing the sql - * @param index used to create dump file name - */ - private static void dumpResult(String sql, int index) { - if (EXPORT_SQL_TYPE_NAME.equalsIgnoreCase(exportType)) { - legalCheck(sql); - } - final String path = targetDirectory + targetFile + index; - try { - SessionDataSet sessionDataSet = session.executeQueryStatement(sql, timeout); - List headers = new ArrayList<>(); - List names = sessionDataSet.getColumnNames(); - List types = sessionDataSet.getColumnTypes(); - if (EXPORT_SQL_TYPE_NAME.equalsIgnoreCase(exportType)) { - writeSqlFile(sessionDataSet, path, names, linesPerFile); - } else { - if (Boolean.TRUE.equals(needDataTypePrinted)) { - for (int i = 0; i < names.size(); i++) { - if (!"Time".equals(names.get(i)) && !"Device".equals(names.get(i))) { - headers.add(String.format("%s(%s)", names.get(i), types.get(i))); - } else { - headers.add(names.get(i)); - } - } - } else { - headers.addAll(names); - } - writeCsvFile(sessionDataSet, path, headers, linesPerFile); - } - sessionDataSet.closeOperationHandle(); - ioTPrinter.println("Export completely!"); - } catch (StatementExecutionException | IoTDBConnectionException | IOException e) { - ioTPrinter.println("Cannot dump result because: " + e.getMessage()); - } - } - - private static void legalCheck(String sql) { - String aggregatePattern = - "\\b(count|sum|avg|extreme|max_value|min_value|first_value|last_value|max_time|min_time|stddev|stddev_pop|stddev_samp|variance|var_pop|var_samp|max_by|min_by)\\b\\s*\\("; - Pattern pattern = Pattern.compile(aggregatePattern, Pattern.CASE_INSENSITIVE); - Matcher matcher = pattern.matcher(sql.toUpperCase(Locale.ROOT)); - if (matcher.find()) { - ioTPrinter.println("The sql you entered is invalid, please don't use aggregate query."); - } - } - - public static String timeTrans(Long time) { - switch (timeFormat) { - case "default": - return RpcUtils.parseLongToDateWithPrecision( - DateTimeFormatter.ISO_OFFSET_DATE_TIME, time, zoneId, timestampPrecision); - case "timestamp": - case "long": - case "number": - return String.valueOf(time); - default: - return ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), zoneId) - .format(DateTimeFormatter.ofPattern(timeFormat)); - } - } - - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - public static void writeCsvFile( - SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) - throws IOException, IoTDBConnectionException, StatementExecutionException { - int fileIndex = 0; - boolean hasNext = true; - while (hasNext) { - int i = 0; - final String finalFilePath = filePath + "_" + fileIndex + ".csv"; - final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); - csvPrinterWrapper.printRecord(headers); - while (i++ < linesPerFile) { - if (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - if (rowRecord.getTimestamp() != 0) { - csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); - } - rowRecord - .getFields() - .forEach( - field -> { - String fieldStringValue = field.getStringValue(); - if (!"null".equals(field.getStringValue())) { - if ((field.getDataType() == TSDataType.TEXT - || field.getDataType() == TSDataType.STRING) - && !fieldStringValue.startsWith("root.")) { - fieldStringValue = "\"" + fieldStringValue + "\""; - } - csvPrinterWrapper.print(fieldStringValue); - } else { - csvPrinterWrapper.print(""); - } - }); - csvPrinterWrapper.println(); - } else { - hasNext = false; - break; - } - } - fileIndex++; - csvPrinterWrapper.flush(); - csvPrinterWrapper.close(); - } - } - - public static void writeSqlFile( - SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) - throws IOException, IoTDBConnectionException, StatementExecutionException { - int fileIndex = 0; - String deviceName = null; - boolean writeNull = false; - List seriesList = new ArrayList<>(headers); - if (CollectionUtils.isEmpty(headers) || headers.size() <= 1) { - writeNull = true; - } else { - if (headers.contains("Device")) { - seriesList.remove("Time"); - seriesList.remove("Device"); - } else { - Path path = new Path(seriesList.get(1), true); - deviceName = path.getDevice(); - seriesList.remove("Time"); - for (int i = 0; i < seriesList.size(); i++) { - String series = seriesList.get(i); - path = new Path(series, true); - seriesList.set(i, path.getMeasurement()); - } - } - } - boolean hasNext = true; - while (hasNext) { - int i = 0; - final String finalFilePath = filePath + "_" + fileIndex + ".sql"; - try (FileWriter writer = new FileWriter(finalFilePath)) { - if (writeNull) { - break; - } - while (i++ < linesPerFile) { - if (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - List headersTemp = new ArrayList<>(seriesList); - List timeseries = new ArrayList<>(); - if (headers.contains("Device")) { - deviceName = fields.get(0).toString(); - if (deviceName.startsWith(SYSTEM_DATABASE + ".")) { - continue; - } - for (String header : headersTemp) { - timeseries.add(deviceName + "." + header); - } - } else { - if (headers.get(1).startsWith(SYSTEM_DATABASE + ".")) { - continue; - } - timeseries.addAll(headers); - timeseries.remove(0); - } - String sqlMiddle = null; - if (Boolean.TRUE.equals(aligned)) { - sqlMiddle = " ALIGNED VALUES (" + rowRecord.getTimestamp() + ","; - } else { - sqlMiddle = " VALUES (" + rowRecord.getTimestamp() + ","; - } - List values = new ArrayList<>(); - if (headers.contains("Device")) { - fields.remove(0); - } - for (int index = 0; index < fields.size(); index++) { - RowRecord next = - session - .executeQueryStatement("SHOW TIMESERIES " + timeseries.get(index), timeout) - .next(); - if (ObjectUtils.isNotEmpty(next)) { - List timeseriesList = next.getFields(); - String value = fields.get(index).toString(); - if (value.equals("null")) { - headersTemp.remove(seriesList.get(index)); - continue; - } - if ("TEXT".equalsIgnoreCase(timeseriesList.get(3).getStringValue())) { - values.add("\"" + value + "\""); - } else { - values.add(value); - } - } else { - headersTemp.remove(seriesList.get(index)); - continue; - } - } - if (CollectionUtils.isNotEmpty(headersTemp)) { - writer.write( - "INSERT INTO " - + deviceName - + "(TIMESTAMP," - + String.join(",", headersTemp) - + ")" - + sqlMiddle - + String.join(",", values) - + ");\n"); - } - - } else { - hasNext = false; - break; - } - } - fileIndex++; - writer.flush(); - } - } - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java deleted file mode 100644 index 9892da625e752..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.type.ExitType; -import org.apache.iotdb.cli.utils.CliContext; -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.cli.utils.JlineUtils; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.lang3.StringUtils; -import org.apache.tsfile.read.common.Field; -import org.apache.tsfile.read.common.RowRecord; -import org.jline.reader.LineReader; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.List; - -import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; - -/** Export Schema CSV file. */ -public class ExportSchema extends AbstractSchemaTool { - - private static final String TARGET_DIR_ARGS = "t"; - private static final String TARGET_DIR_ARGS_NAME = "target"; - private static final String TARGET_DIR_NAME = "targetDir"; - - private static final String TARGET_PATH_ARGS = "path"; - private static final String TARGET_PATH_ARGS_NAME = "path_pattern"; - private static final String TARGET_PATH_NAME = "exportPathPattern"; - private static String queryPath; - - private static final String TARGET_FILE_ARGS = "pf"; - private static final String TARGET_FILE_ARGS_NAME = "path_pattern_file"; - private static final String TARGET_FILE_NAME = "exportPathPatternFile"; - - private static final String LINES_PER_FILE_ARGS = "lpf"; - private static final String LINES_PER_FILE_ARGS_NAME = "lines_per_file"; - private static final String LINES_PER_FILE_NAME = "linesPerFile"; - private static int linesPerFile = 10000; - - private static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema"; - - private static final String DUMP_FILE_NAME_DEFAULT = "dump"; - private static String targetFile = DUMP_FILE_NAME_DEFAULT; - - private static String targetDirectory; - - private static long timeout = 60000; - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - - private static final String BASE_VIEW_TYPE = "BASE"; - private static final String HEADER_VIEW_TYPE = "ViewType"; - private static final String HEADER_TIMESERIES = "Timeseries"; - - @SuppressWarnings({ - "squid:S3776", - "squid:S2093" - }) // Suppress high Cognitive Complexity warning, ignore try-with-resources - /* main function of export csv tool. */ - public static void main(String[] args) { - Options options = createOptions(); - HelpFormatter hf = new HelpFormatter(); - CommandLine commandLine = null; - CommandLineParser parser = new DefaultParser(); - hf.setOptionComparator(null); // avoid reordering - hf.setWidth(MAX_HELP_CONSOLE_WIDTH); - - if (args == null || args.length == 0) { - ioTPrinter.println("Too few params input, please check the following hint."); - hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - try { - commandLine = parser.parse(options, args); - } catch (ParseException e) { - ioTPrinter.println(e.getMessage()); - hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - if (commandLine.hasOption(HELP_ARGS)) { - hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - int exitCode = CODE_OK; - try { - parseBasicParams(commandLine); - parseSpecialParams(commandLine); - session = new Session(host, Integer.parseInt(port), username, password); - session.open(false); - if (queryPath == null) { - String pathFile = commandLine.getOptionValue(TARGET_FILE_ARGS); - String path; - if (pathFile == null) { - LineReader lineReader = - JlineUtils.getLineReader( - new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), - username, - host, - port); - path = lineReader.readLine(EXPORT_SCHEMA_CLI_PREFIX + "> please input path pattern: "); - ioTPrinter.println(path); - String[] values = path.trim().split(";"); - for (int i = 0; i < values.length; i++) { - if (StringUtils.isBlank(values[i])) { - continue; - } else { - dumpResult(values[i], i); - } - } - } else if (!pathFile.endsWith(".txt")) { - ioTPrinter.println("The file name must end with \"txt\"!"); - hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } else { - dumpFromPathFile(pathFile); - } - } else { - dumpResult(queryPath, 0); - } - } catch (IOException e) { - ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (ArgsErrorException e) { - ioTPrinter.println("Invalid args: " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (IoTDBConnectionException e) { - ioTPrinter.println("Connect failed because " + e.getMessage()); - exitCode = CODE_ERROR; - } finally { - if (session != null) { - try { - session.close(); - } catch (IoTDBConnectionException e) { - exitCode = CODE_ERROR; - ioTPrinter.println( - "Encounter an error when closing session, error is: " + e.getMessage()); - } - } - } - System.exit(exitCode); - } - - private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { - targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_ARGS_NAME, commandLine, null); - queryPath = commandLine.getOptionValue(TARGET_PATH_ARGS); - String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS); - if (timeoutString != null) { - timeout = Long.parseLong(timeoutString); - } - if (targetFile == null) { - targetFile = DUMP_FILE_NAME_DEFAULT; - } - if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { - targetDirectory += File.separator; - } - if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) { - linesPerFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS)); - } - } - - /** - * commandline option create. - * - * @return object Options - */ - private static Options createOptions() { - Options options = createNewOptions(); - - Option opTargetFile = - Option.builder(TARGET_DIR_ARGS) - .required() - .longOpt(TARGET_DIR_ARGS_NAME) - .hasArg() - .argName(TARGET_DIR_NAME) - .desc("Target File Directory (required)") - .build(); - options.addOption(opTargetFile); - - Option targetPathPattern = - Option.builder(TARGET_PATH_ARGS) - .longOpt(TARGET_PATH_ARGS_NAME) - .hasArg() - .argName(TARGET_PATH_NAME) - .desc("Export Path Pattern (optional)") - .build(); - options.addOption(targetPathPattern); - - Option targetFileName = - Option.builder(TARGET_FILE_ARGS) - .longOpt(TARGET_FILE_ARGS_NAME) - .hasArg() - .argName(TARGET_FILE_NAME) - .desc("Export File Name (optional)") - .build(); - options.addOption(targetFileName); - - Option opLinesPerFile = - Option.builder(LINES_PER_FILE_ARGS) - .longOpt(LINES_PER_FILE_ARGS_NAME) - .hasArg() - .argName(LINES_PER_FILE_NAME) - .desc("Lines per dump file.") - .build(); - options.addOption(opLinesPerFile); - - Option opTimeout = - Option.builder(TIMEOUT_ARGS) - .longOpt(TIMEOUT_ARGS_NAME) - .hasArg() - .argName(TIMEOUT_ARGS) - .desc(timeout + " Timeout for session query") - .build(); - options.addOption(opTimeout); - - Option opHelp = - Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help information").build(); - options.addOption(opHelp); - return options; - } - - /** - * This method will be called, if the query commands are written in a sql file. - * - * @param pathFile sql file path - * @throws IOException exception - */ - private static void dumpFromPathFile(String pathFile) throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader(pathFile))) { - String path; - int index = 0; - while ((path = reader.readLine()) != null) { - dumpResult(path, index); - index++; - } - } - } - - /** - * Dump files from database to CSV file. - * - * @param pattern used to be export schema - * @param index used to create dump file name - */ - private static void dumpResult(String pattern, int index) { - File file = new File(targetDirectory); - if (!file.isDirectory()) { - file.mkdir(); - } - final String path = targetDirectory + targetFile + index; - try { - SessionDataSet sessionDataSet = - session.executeQueryStatement("show timeseries " + pattern, timeout); - writeCsvFile(sessionDataSet, path, sessionDataSet.getColumnNames(), linesPerFile); - sessionDataSet.closeOperationHandle(); - ioTPrinter.println("Export completely!"); - } catch (StatementExecutionException | IoTDBConnectionException | IOException e) { - ioTPrinter.println("Cannot dump result because: " + e.getMessage()); - } - } - - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - public static void writeCsvFile( - SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) - throws IOException, IoTDBConnectionException, StatementExecutionException { - int viewTypeIndex = headers.indexOf(HEADER_VIEW_TYPE); - int timeseriesIndex = headers.indexOf(HEADER_TIMESERIES); - - int fileIndex = 0; - boolean hasNext = true; - while (hasNext) { - int i = 0; - final String finalFilePath = filePath + "_" + fileIndex + ".csv"; - final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); - while (i++ < linesPerFile) { - if (sessionDataSet.hasNext()) { - if (i == 1) { - csvPrinterWrapper.printRecord(HEAD_COLUMNS); - } - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE + ".") - || !fields.get(viewTypeIndex).getStringValue().equals(BASE_VIEW_TYPE)) { - continue; - } - HEAD_COLUMNS.forEach( - column -> { - Field field = fields.get(headers.indexOf(column)); - String fieldStringValue = field.getStringValue(); - if (!"null".equals(field.getStringValue())) { - csvPrinterWrapper.print(fieldStringValue); - } else { - csvPrinterWrapper.print(""); - } - }); - csvPrinterWrapper.println(); - } else { - hasNext = false; - break; - } - } - fileIndex++; - csvPrinterWrapper.flush(); - csvPrinterWrapper.close(); - } - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportTsFile.java deleted file mode 100644 index 723e410741e54..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportTsFile.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.type.ExitType; -import org.apache.iotdb.cli.utils.CliContext; -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.cli.utils.JlineUtils; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.ParseException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.fileSystem.FSFactoryProducer; -import org.apache.tsfile.read.common.Field; -import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.common.RowRecord; -import org.apache.tsfile.write.TsFileWriter; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.jline.reader.LineReader; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class ExportTsFile extends AbstractTsFileTool { - - private static final String TARGET_DIR_ARGS = "t"; - private static final String TARGET_DIR_NAME = "targetDirectory"; - private static final String TARGET_FILE_ARGS = "tfn"; - private static final String TARGET_FILE_NAME = "targetFileName"; - - private static final String SQL_FILE_ARGS = "s"; - private static final String SQL_FILE_NAME = "sourceSqlFile"; - private static final String QUERY_COMMAND_ARGS = "q"; - private static final String QUERY_COMMAND_NAME = "queryCommand"; - private static final String DUMP_FILE_NAME_DEFAULT = "dump"; - private static final String TSFILEDB_CLI_PREFIX = "ExportTsFile"; - - private static Session session; - - private static String targetDirectory; - private static String targetFile = DUMP_FILE_NAME_DEFAULT; - private static String queryCommand; - - private static long timeout = -1; - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - - @SuppressWarnings({ - "squid:S3776", - "squid:S2093" - }) // Suppress high Cognitive Complexity warning, ignore try-with-resources - /* main function of export tsFile tool. */ - public static void main(String[] args) { - createOptions(); - HelpFormatter hf = new HelpFormatter(); - CommandLine commandLine = null; - CommandLineParser parser = new DefaultParser(); - hf.setOptionComparator(null); // avoid reordering - hf.setWidth(MAX_HELP_CONSOLE_WIDTH); - - if (args == null || args.length == 0) { - ioTPrinter.println("Too few params input, please check the following hint."); - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - - try { - commandLine = parser.parse(options, args); - } catch (ParseException e) { - ioTPrinter.println(e.getMessage()); - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - if (commandLine.hasOption(HELP_ARGS)) { - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - - int exitCode = CODE_OK; - try { - parseBasicParams(commandLine); - parseSpecialParams(commandLine); - - session = new Session(host, Integer.parseInt(port), username, password); - session.open(false); - - if (queryCommand == null) { - String sqlFile = commandLine.getOptionValue(SQL_FILE_ARGS); - String sql; - - if (sqlFile == null) { - LineReader lineReader = - JlineUtils.getLineReader( - new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), - username, - host, - port); - sql = lineReader.readLine(TSFILEDB_CLI_PREFIX + "> please input query: "); - ioTPrinter.println(sql); - String[] values = sql.trim().split(";"); - for (int i = 0; i < values.length; i++) { - legalCheck(values[i]); - dumpResult(values[i], i); - } - - } else { - dumpFromSqlFile(sqlFile); - } - } else { - legalCheck(queryCommand); - dumpResult(queryCommand, 0); - } - - } catch (IOException e) { - ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (ArgsErrorException e) { - ioTPrinter.println("Invalid args: " + e.getMessage()); - exitCode = CODE_ERROR; - } catch (IoTDBConnectionException e) { - ioTPrinter.println("Connect failed because " + e.getMessage()); - exitCode = CODE_ERROR; - } finally { - if (session != null) { - try { - session.close(); - } catch (IoTDBConnectionException e) { - exitCode = CODE_ERROR; - ioTPrinter.println( - "Encounter an error when closing session, error is: " + e.getMessage()); - } - } - } - System.exit(exitCode); - } - - private static void legalCheck(String sql) { - String sqlLower = sql.toLowerCase(); - if (sqlLower.contains("count(") - || sqlLower.contains("sum(") - || sqlLower.contains("avg(") - || sqlLower.contains("extreme(") - || sqlLower.contains("max_value(") - || sqlLower.contains("min_value(") - || sqlLower.contains("first_value(") - || sqlLower.contains("last_value(") - || sqlLower.contains("max_time(") - || sqlLower.contains("min_time(") - || sqlLower.contains("stddev(") - || sqlLower.contains("stddev_pop(") - || sqlLower.contains("stddev_samp(") - || sqlLower.contains("variance(") - || sqlLower.contains("var_pop(") - || sqlLower.contains("var_samp(") - || sqlLower.contains("max_by(") - || sqlLower.contains("min_by(")) { - ioTPrinter.println("The sql you entered is invalid, please don't use aggregate query."); - System.exit(CODE_ERROR); - } - } - - private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { - targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_NAME, commandLine); - queryCommand = commandLine.getOptionValue(QUERY_COMMAND_ARGS); - targetFile = commandLine.getOptionValue(TARGET_FILE_ARGS); - String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS); - if (timeoutString != null) { - timeout = Long.parseLong(timeoutString); - } - if (targetFile == null) { - targetFile = DUMP_FILE_NAME_DEFAULT; - } - - if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { - targetDirectory += File.separator; - } - } - - /** - * commandline option create. - * - * @return object Options - */ - private static void createOptions() { - createBaseOptions(); - - Option opTargetFile = - Option.builder(TARGET_DIR_ARGS) - .required() - .argName(TARGET_DIR_NAME) - .hasArg() - .desc("Target File Directory (required)") - .build(); - options.addOption(opTargetFile); - - Option targetFileName = - Option.builder(TARGET_FILE_ARGS) - .argName(TARGET_FILE_NAME) - .hasArg() - .desc("Export file name (optional)") - .build(); - options.addOption(targetFileName); - - Option opSqlFile = - Option.builder(SQL_FILE_ARGS) - .argName(SQL_FILE_NAME) - .hasArg() - .desc("SQL File Path (optional)") - .build(); - options.addOption(opSqlFile); - - Option opQuery = - Option.builder(QUERY_COMMAND_ARGS) - .argName(QUERY_COMMAND_NAME) - .hasArg() - .desc("The query command that you want to execute. (optional)") - .build(); - options.addOption(opQuery); - - Option opHelp = - Option.builder(HELP_ARGS) - .longOpt(HELP_ARGS) - .hasArg(false) - .desc("Display help information") - .build(); - options.addOption(opHelp); - - Option opTimeout = - Option.builder(TIMEOUT_ARGS) - .argName(TIMEOUT_NAME) - .hasArg() - .desc("Timeout for session query") - .build(); - options.addOption(opTimeout); - } - - /** - * This method will be called, if the query commands are written in a sql file. - * - * @param filePath:file path - * @throws IOException: exception - */ - private static void dumpFromSqlFile(String filePath) throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - String sql; - int i = 0; - while ((sql = reader.readLine()) != null) { - legalCheck(sql); - dumpResult(sql, i++); - } - } - } - - /** - * Dump files from database to tsFile. - * - * @param sql export the result of executing the sql - */ - private static void dumpResult(String sql, int index) { - final String path = targetDirectory + targetFile + index + ".tsfile"; - try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql, timeout)) { - long start = System.currentTimeMillis(); - writeTsFileFile(sessionDataSet, path); - long end = System.currentTimeMillis(); - ioTPrinter.println("Export completely!cost: " + (end - start) + " ms."); - } catch (StatementExecutionException - | IoTDBConnectionException - | IOException - | WriteProcessException e) { - ioTPrinter.println("Cannot dump result because: " + e.getMessage()); - } - } - - @SuppressWarnings({ - "squid:S3776", - "squid:S6541" - }) // Suppress high Cognitive Complexity warning, Suppress many task in one method warning - public static void writeTsFileFile(SessionDataSet sessionDataSet, String filePath) - throws IOException, - IoTDBConnectionException, - StatementExecutionException, - WriteProcessException { - List columnNames = sessionDataSet.getColumnNames(); - List columnTypes = sessionDataSet.getColumnTypes(); - File f = FSFactoryProducer.getFSFactory().getFile(filePath); - if (f.exists()) { - Files.delete(f.toPath()); - } - HashSet deviceFilterSet = new HashSet<>(); - try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { - Map> schemaMap = new LinkedHashMap<>(); - for (int i = 0; i < columnNames.size(); i++) { - String column = columnNames.get(i); - if (!column.startsWith("root.")) { - continue; - } - TSDataType tsDataType = getTsDataType(columnTypes.get(i)); - Path path = new Path(column, true); - String deviceId = path.getDevice(); - try (SessionDataSet deviceDataSet = - session.executeQueryStatement("show devices " + deviceId, timeout)) { - List deviceList = deviceDataSet.next().getFields(); - if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { - deviceFilterSet.add(deviceId); - } - } - MeasurementSchema measurementSchema = - new MeasurementSchema(path.getMeasurement(), tsDataType); - - List seriesList = - session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); - - measurementSchema.setEncoding( - TSEncoding.valueOf(seriesList.get(4).getStringValue()).serialize()); - measurementSchema.setCompressor( - CompressionType.valueOf(seriesList.get(5).getStringValue()).serialize()); - schemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); - } - List tabletList = new ArrayList<>(); - for (Map.Entry> stringListEntry : schemaMap.entrySet()) { - String deviceId = stringListEntry.getKey(); - List schemaList = stringListEntry.getValue(); - Tablet tablet = new Tablet(deviceId, schemaList); - tablet.initBitMaps(); - Path path = new Path(tablet.deviceId); - if (deviceFilterSet.contains(tablet.deviceId)) { - tsFileWriter.registerAlignedTimeseries(path, schemaList); - } else { - tsFileWriter.registerTimeseries(path, schemaList); - } - tabletList.add(tablet); - } - if (tabletList.isEmpty()) { - ioTPrinter.println("!!!Warning:Tablet is empty,no data can be exported."); - System.exit(CODE_ERROR); - } - while (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - int i = 0; - while (i < fields.size()) { - for (Tablet tablet : tabletList) { - int rowIndex = tablet.rowSize++; - tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); - List schemas = tablet.getSchemas(); - for (int j = 0; j < schemas.size(); j++) { - MeasurementSchema measurementSchema = schemas.get(j); - Object value = fields.get(i).getObjectValue(measurementSchema.getType()); - if (value == null) { - tablet.bitMaps[j].mark(rowIndex); - } - tablet.addValue(measurementSchema.getMeasurementId(), rowIndex, value); - i++; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - writeToTsfile(deviceFilterSet, tsFileWriter, tablet); - tablet.initBitMaps(); - tablet.reset(); - } - } - } - } - for (Tablet tablet : tabletList) { - if (tablet.rowSize != 0) { - writeToTsfile(deviceFilterSet, tsFileWriter, tablet); - } - } - tsFileWriter.flushAllChunkGroups(); - } - } - - private static void writeToTsfile( - HashSet deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) - throws IOException, WriteProcessException { - if (deviceFilterSet.contains(tablet.deviceId)) { - tsFileWriter.writeAligned(tablet); - } else { - tsFileWriter.write(tablet); - } - } - - private static TSDataType getTsDataType(String type) { - return TSDataType.valueOf(type); - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java deleted file mode 100644 index a16cce67a2ed2..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java +++ /dev/null @@ -1,1069 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; -import org.apache.iotdb.db.utils.DateTimeUtils; -import org.apache.iotdb.db.utils.constant.SqlConstant; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.thrift.annotation.Nullable; -import org.apache.tsfile.common.constant.TsFileConstant; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.Field; -import org.apache.tsfile.read.common.RowRecord; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.tsfile.enums.TSDataType.STRING; -import static org.apache.tsfile.enums.TSDataType.TEXT; - -public class ImportData extends AbstractDataTool { - - private static final String FILE_ARGS = "s"; - private static final String FILE_NAME = "sourceFileOrFolder"; - - private static final String FAILED_FILE_ARGS = "fd"; - private static final String FAILED_FILE_NAME = "failed file directory"; - - private static final String BATCH_POINT_SIZE_ARGS = "batch"; - private static final String BATCH_POINT_SIZE_NAME = "batch point size"; - - private static final String ALIGNED_ARGS = "aligned"; - private static final String ALIGNED_NAME = "use the aligned interface"; - - private static final String CSV_SUFFIXS = "csv"; - private static final String TXT_SUFFIXS = "txt"; - - private static final String SQL_SUFFIXS = "sql"; - - private static final String TIMESTAMP_PRECISION_ARGS = "tp"; - private static final String TIMESTAMP_PRECISION_NAME = "timestamp precision (ms/us/ns)"; - - private static final String TYPE_INFER_ARGS = "typeInfer"; - private static final String TYPE_INFER_ARGS_NAME = "type infer"; - - private static final String LINES_PER_FAILED_FILE_ARGS = "lpf"; - private static final String LINES_PER_FAILED_FILE_ARGS_NAME = "linesPerFailedFile"; - - private static final String TSFILEDB_CLI_PREFIX = "ImportData"; - - private static String targetPath; - private static String failedFileDirectory = null; - private static int linesPerFailedFile = 10000; - private static Boolean aligned = false; - - private static String timeColumn = "Time"; - private static String deviceColumn = "Device"; - - private static int batchPointSize = 100_000; - - private static String timestampPrecision = "ms"; - - private static final String DATATYPE_BOOLEAN = "boolean"; - private static final String DATATYPE_INT = "int"; - private static final String DATATYPE_LONG = "long"; - private static final String DATATYPE_FLOAT = "float"; - private static final String DATATYPE_DOUBLE = "double"; - private static final String DATATYPE_TIMESTAMP = "timestamp"; - private static final String DATATYPE_DATE = "date"; - private static final String DATATYPE_BLOB = "blob"; - private static final String DATATYPE_NAN = "NaN"; - private static final String DATATYPE_TEXT = "text"; - - private static final String DATATYPE_NULL = "null"; - private static int fetchSize = 1000; - - private static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because "; - - private static final Map TYPE_INFER_KEY_DICT = new HashMap<>(); - - static { - TYPE_INFER_KEY_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); - TYPE_INFER_KEY_DICT.put(DATATYPE_INT, TSDataType.FLOAT); - TYPE_INFER_KEY_DICT.put(DATATYPE_LONG, TSDataType.DOUBLE); - TYPE_INFER_KEY_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); - TYPE_INFER_KEY_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); - TYPE_INFER_KEY_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); - TYPE_INFER_KEY_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); - TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.TEXT); - TYPE_INFER_KEY_DICT.put(DATATYPE_NAN, TSDataType.DOUBLE); - } - - private static final Map TYPE_INFER_VALUE_DICT = new HashMap<>(); - - static { - TYPE_INFER_VALUE_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); - TYPE_INFER_VALUE_DICT.put(DATATYPE_INT, TSDataType.INT32); - TYPE_INFER_VALUE_DICT.put(DATATYPE_LONG, TSDataType.INT64); - TYPE_INFER_VALUE_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); - TYPE_INFER_VALUE_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); - TYPE_INFER_VALUE_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); - TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); - TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.TEXT); - TYPE_INFER_VALUE_DICT.put(DATATYPE_TEXT, TSDataType.TEXT); - } - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - - /** - * create the commandline options. - * - * @return object Options - */ - private static Options createOptions() { - Options options = createNewOptions(); - - Option opFile = - Option.builder(FILE_ARGS) - .required() - .argName(FILE_NAME) - .hasArg() - .desc( - "If input a file path, load a csv file, " - + "otherwise load all csv file under this directory (required)") - .build(); - options.addOption(opFile); - - Option opFailedFile = - Option.builder(FAILED_FILE_ARGS) - .argName(FAILED_FILE_NAME) - .hasArg() - .desc( - "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)") - .build(); - options.addOption(opFailedFile); - - Option opAligned = - Option.builder(ALIGNED_ARGS) - .argName(ALIGNED_NAME) - .hasArg() - .desc("Whether to use the interface of aligned(only csv optional)") - .build(); - options.addOption(opAligned); - - Option opHelp = - Option.builder(HELP_ARGS) - .longOpt(HELP_ARGS) - .hasArg(false) - .desc("Display help information") - .build(); - options.addOption(opHelp); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); - - Option opBatchPointSize = - Option.builder(BATCH_POINT_SIZE_ARGS) - .argName(BATCH_POINT_SIZE_NAME) - .hasArg() - .desc("100000 (optional)") - .build(); - options.addOption(opBatchPointSize); - - Option opTimestampPrecision = - Option.builder(TIMESTAMP_PRECISION_ARGS) - .argName(TIMESTAMP_PRECISION_NAME) - .hasArg() - .desc("Timestamp precision (ms/us/ns)") - .build(); - - options.addOption(opTimestampPrecision); - - Option opTypeInfer = - Option.builder(TYPE_INFER_ARGS) - .argName(TYPE_INFER_ARGS_NAME) - .numberOfArgs(5) - .hasArgs() - .valueSeparator(',') - .desc("Define type info by option:\"boolean=text,int=long, ...") - .build(); - options.addOption(opTypeInfer); - - Option opFailedLinesPerFile = - Option.builder(LINES_PER_FAILED_FILE_ARGS) - .argName(LINES_PER_FAILED_FILE_ARGS_NAME) - .hasArgs() - .desc("Lines per failed file") - .build(); - options.addOption(opFailedLinesPerFile); - - return options; - } - - /** - * parse optional params - * - * @param commandLine - */ - private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { - timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS); - targetPath = commandLine.getOptionValue(FILE_ARGS); - if (commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS) != null) { - batchPointSize = Integer.parseInt(commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS)); - } - if (commandLine.getOptionValue(FAILED_FILE_ARGS) != null) { - failedFileDirectory = commandLine.getOptionValue(FAILED_FILE_ARGS); - File file = new File(failedFileDirectory); - if (!file.isDirectory()) { - file.mkdir(); - failedFileDirectory = file.getAbsolutePath() + File.separator; - } else if (!failedFileDirectory.endsWith("/") && !failedFileDirectory.endsWith("\\")) { - failedFileDirectory += File.separator; - } - } - if (commandLine.getOptionValue(ALIGNED_ARGS) != null) { - aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS)); - } - - if (commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS) != null) { - timestampPrecision = commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS); - } - final String[] opTypeInferValues = commandLine.getOptionValues(TYPE_INFER_ARGS); - if (opTypeInferValues != null && opTypeInferValues.length > 0) { - for (String opTypeInferValue : opTypeInferValues) { - if (opTypeInferValue.contains("=")) { - final String[] typeInfoExpressionArr = opTypeInferValue.split("="); - final String key = typeInfoExpressionArr[0]; - final String value = typeInfoExpressionArr[1]; - applyTypeInferArgs(key, value); - } - } - } - if (commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS) != null) { - linesPerFailedFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS)); - } - } - - private static void applyTypeInferArgs(String key, String value) throws ArgsErrorException { - if (!TYPE_INFER_KEY_DICT.containsKey(key)) { - throw new ArgsErrorException("Unknown type infer key: " + key); - } - if (!TYPE_INFER_VALUE_DICT.containsKey(value)) { - throw new ArgsErrorException("Unknown type infer value: " + value); - } - if (key.equals(DATATYPE_NAN) - && !(value.equals(DATATYPE_FLOAT) - || value.equals(DATATYPE_DOUBLE) - || value.equals(DATATYPE_TEXT))) { - throw new ArgsErrorException("NaN can not convert to " + value); - } - if (key.equals(DATATYPE_BOOLEAN) - && !(value.equals(DATATYPE_BOOLEAN) || value.equals(DATATYPE_TEXT))) { - throw new ArgsErrorException("Boolean can not convert to " + value); - } - final TSDataType srcType = TYPE_INFER_VALUE_DICT.get(key); - final TSDataType dstType = TYPE_INFER_VALUE_DICT.get(value); - if (dstType.getType() < srcType.getType()) { - throw new ArgsErrorException(key + " can not convert to " + value); - } - TYPE_INFER_KEY_DICT.put(key, TYPE_INFER_VALUE_DICT.get(value)); - } - - public static void main(String[] args) throws IoTDBConnectionException { - Options options = createOptions(); - HelpFormatter hf = new HelpFormatter(); - hf.setOptionComparator(null); - hf.setWidth(MAX_HELP_CONSOLE_WIDTH); - CommandLine commandLine = null; - CommandLineParser parser = new DefaultParser(); - - if (args == null || args.length == 0) { - ioTPrinter.println("Too few params input, please check the following hint."); - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - try { - commandLine = parser.parse(options, args); - } catch (org.apache.commons.cli.ParseException e) { - ioTPrinter.println("Parse error: " + e.getMessage()); - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - if (commandLine.hasOption(HELP_ARGS)) { - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - - try { - parseBasicParams(commandLine); - String filename = commandLine.getOptionValue(FILE_ARGS); - if (filename == null) { - hf.printHelp(TSFILEDB_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - parseSpecialParams(commandLine); - } catch (ArgsErrorException e) { - ioTPrinter.println("Args error: " + e.getMessage()); - System.exit(CODE_ERROR); - } catch (Exception e) { - ioTPrinter.println("Encounter an error, because: " + e.getMessage()); - System.exit(CODE_ERROR); - } - - System.exit( - importFromTargetPath( - host, Integer.parseInt(port), username, password, targetPath, timeZoneID)); - } - - /** - * Specifying a CSV file or a directory including CSV files that you want to import. This method - * can be offered to console cli to implement importing CSV file by command. - * - * @param host - * @param port - * @param username - * @param password - * @param targetPath a CSV file or a directory including CSV files - * @param timeZone - * @return the status code - * @throws IoTDBConnectionException - */ - @SuppressWarnings({"squid:S2093"}) // ignore try-with-resources - public static int importFromTargetPath( - String host, int port, String username, String password, String targetPath, String timeZone) - throws IoTDBConnectionException { - try { - session = new Session(host, port, username, password, false); - session.open(false); - timeZoneID = timeZone; - setTimeZone(); - - File file = new File(targetPath); - if (file.isFile()) { - if (file.getName().endsWith(SQL_SUFFIXS)) { - importFromSqlFile(file); - } else { - importFromSingleFile(file); - } - } else if (file.isDirectory()) { - File[] files = file.listFiles(); - if (files == null) { - return CODE_OK; - } - - for (File subFile : files) { - if (subFile.isFile()) { - if (subFile.getName().endsWith(SQL_SUFFIXS)) { - importFromSqlFile(subFile); - } else { - importFromSingleFile(subFile); - } - } - } - } else { - ioTPrinter.println("File not found!"); - return CODE_ERROR; - } - } catch (IoTDBConnectionException | StatementExecutionException e) { - ioTPrinter.println("Encounter an error when connecting to server, because " + e.getMessage()); - return CODE_ERROR; - } finally { - if (session != null) { - session.close(); - } - } - return CODE_OK; - } - - /** - * import the CSV file and load headers and records. - * - * @param file the File object of the CSV file that you want to import. - */ - private static void importFromSingleFile(File file) { - if (file.getName().endsWith(CSV_SUFFIXS) || file.getName().endsWith(TXT_SUFFIXS)) { - try { - CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); - List headerNames = csvRecords.getHeaderNames(); - Stream records = csvRecords.stream(); - if (headerNames.isEmpty()) { - ioTPrinter.println("Empty file!"); - return; - } - if (!timeColumn.equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { - ioTPrinter.println("The first field of header must be `Time`!"); - return; - } - String failedFilePath = null; - if (failedFileDirectory == null) { - failedFilePath = file.getAbsolutePath() + ".failed"; - } else { - failedFilePath = failedFileDirectory + file.getName() + ".failed"; - } - if (!deviceColumn.equalsIgnoreCase(headerNames.get(1))) { - writeDataAlignedByTime(headerNames, records, failedFilePath); - } else { - writeDataAlignedByDevice(headerNames, records, failedFilePath); - } - } catch (IOException | IllegalPathException e) { - ioTPrinter.println("CSV file read exception because: " + e.getMessage()); - } - } else { - ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); - } - } - - @SuppressWarnings("java:S2259") - private static void importFromSqlFile(File file) { - ArrayList> failedRecords = new ArrayList<>(); - String failedFilePath = null; - if (failedFileDirectory == null) { - failedFilePath = file.getAbsolutePath() + ".failed"; - } else { - failedFilePath = failedFileDirectory + file.getName() + ".failed"; - } - try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { - String sql; - while ((sql = br.readLine()) != null) { - try { - session.executeNonQueryStatement(sql); - } catch (IoTDBConnectionException | StatementExecutionException e) { - failedRecords.add(Arrays.asList(sql)); - } - } - ioTPrinter.println(file.getName() + " Import completely!"); - } catch (IOException e) { - ioTPrinter.println("SQL file read exception because: " + e.getMessage()); - } - if (!failedRecords.isEmpty()) { - FileWriter writer = null; - try { - writer = new FileWriter(failedFilePath); - for (List failedRecord : failedRecords) { - writer.write(failedRecord.get(0).toString() + "\n"); - } - } catch (IOException e) { - ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); - } finally { - if (ObjectUtils.isNotEmpty(writer)) { - try { - writer.flush(); - writer.close(); - } catch (IOException e) { - } - } - } - } - } - - /** - * if the data is aligned by time, the data will be written by this method. - * - * @param headerNames the header names of CSV file - * @param records the records of CSV file - * @param failedFilePath the directory to save the failed files - */ - @SuppressWarnings("squid:S3776") - private static void writeDataAlignedByTime( - List headerNames, Stream records, String failedFilePath) - throws IllegalPathException { - HashMap> deviceAndMeasurementNames = new HashMap<>(); - HashMap headerTypeMap = new HashMap<>(); - HashMap headerNameMap = new HashMap<>(); - parseHeaders(headerNames, deviceAndMeasurementNames, headerTypeMap, headerNameMap); - - Set devices = deviceAndMeasurementNames.keySet(); - if (headerTypeMap.isEmpty()) { - queryType(devices, headerTypeMap, "Time"); - } - - List deviceIds = new ArrayList<>(); - List times = new ArrayList<>(); - List> measurementsList = new ArrayList<>(); - List> typesList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); - - AtomicReference hasStarted = new AtomicReference<>(false); - AtomicInteger pointSize = new AtomicInteger(0); - - ArrayList> failedRecords = new ArrayList<>(); - - records.forEach( - recordObj -> { - if (Boolean.FALSE.equals(hasStarted.get())) { - hasStarted.set(true); - } else if (pointSize.get() >= batchPointSize) { - writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - - boolean isFail = false; - - for (Map.Entry> entry : deviceAndMeasurementNames.entrySet()) { - String deviceId = entry.getKey(); - List measurementNames = entry.getValue(); - ArrayList types = new ArrayList<>(); - ArrayList values = new ArrayList<>(); - ArrayList measurements = new ArrayList<>(); - for (String measurement : measurementNames) { - String header = deviceId + "." + measurement; - String value = recordObj.get(headerNameMap.get(header)); - if (!"".equals(value)) { - TSDataType type; - if (!headerTypeMap.containsKey(header)) { - type = typeInfer(value); - if (type != null) { - headerTypeMap.put(header, type); - } else { - ioTPrinter.printf( - "Line '%s', column '%s': '%s' unknown type%n", - recordObj.getRecordNumber(), header, value); - isFail = true; - } - } - type = headerTypeMap.get(header); - if (type != null) { - Object valueTrans = typeTrans(value, type); - if (valueTrans == null) { - isFail = true; - ioTPrinter.printf( - "Line '%s', column '%s': '%s' can't convert to '%s'%n", - recordObj.getRecordNumber(), header, value, type); - } else { - measurements.add(header.replace(deviceId + '.', "")); - types.add(type); - values.add(valueTrans); - pointSize.getAndIncrement(); - } - } - } - } - if (!measurements.isEmpty()) { - times.add(parseTimestamp(recordObj.get(timeColumn))); - deviceIds.add(deviceId); - typesList.add(types); - valuesList.add(values); - measurementsList.add(measurements); - } - } - if (isFail) { - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - } - }); - if (!deviceIds.isEmpty()) { - writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - - if (!failedRecords.isEmpty()) { - writeFailedLinesFile(headerNames, failedFilePath, failedRecords); - } - if (Boolean.TRUE.equals(hasStarted.get())) { - ioTPrinter.println("Import completely!"); - } else { - ioTPrinter.println("No records!"); - } - } - - /** - * if the data is aligned by device, the data will be written by this method. - * - * @param headerNames the header names of CSV file - * @param records the records of CSV file - * @param failedFilePath the directory to save the failed files - */ - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - private static void writeDataAlignedByDevice( - List headerNames, Stream records, String failedFilePath) - throws IllegalPathException { - HashMap headerTypeMap = new HashMap<>(); - HashMap headerNameMap = new HashMap<>(); - parseHeaders(headerNames, null, headerTypeMap, headerNameMap); - - AtomicReference deviceName = new AtomicReference<>(null); - - HashSet typeQueriedDevice = new HashSet<>(); - - // the data that interface need - List times = new ArrayList<>(); - List> typesList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); - List> measurementsList = new ArrayList<>(); - - AtomicInteger pointSize = new AtomicInteger(0); - - ArrayList> failedRecords = new ArrayList<>(); - - records.forEach( - recordObj -> { - // only run in first record - if (deviceName.get() == null) { - deviceName.set(recordObj.get(1)); - } else if (!Objects.equals(deviceName.get(), recordObj.get(1))) { - // if device changed - writeAndEmptyDataSet( - deviceName.get(), times, typesList, valuesList, measurementsList, 3); - deviceName.set(recordObj.get(1)); - pointSize.set(0); - } else if (pointSize.get() >= batchPointSize) { - // insert a batch - writeAndEmptyDataSet( - deviceName.get(), times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - - // the data of the record - ArrayList types = new ArrayList<>(); - ArrayList values = new ArrayList<>(); - ArrayList measurements = new ArrayList<>(); - - AtomicReference isFail = new AtomicReference<>(false); - - // read data from record - for (Map.Entry headerNameEntry : headerNameMap.entrySet()) { - // headerNameWithoutType is equal to headerName if the CSV column do not have data type. - String headerNameWithoutType = headerNameEntry.getKey(); - String headerName = headerNameEntry.getValue(); - String value = recordObj.get(headerName); - if (!"".equals(value)) { - TSDataType type; - // Get the data type directly if the CSV column have data type. - if (!headerTypeMap.containsKey(headerNameWithoutType)) { - boolean hasResult = false; - // query the data type in iotdb - if (!typeQueriedDevice.contains(deviceName.get())) { - if (headerTypeMap.isEmpty()) { - Set devices = new HashSet<>(); - devices.add(deviceName.get()); - queryType(devices, headerTypeMap, deviceColumn); - } - typeQueriedDevice.add(deviceName.get()); - } - type = typeInfer(value); - if (type != null) { - headerTypeMap.put(headerNameWithoutType, type); - } else { - ioTPrinter.printf( - "Line '%s', column '%s': '%s' unknown type%n", - recordObj.getRecordNumber(), headerNameWithoutType, value); - isFail.set(true); - } - } - type = headerTypeMap.get(headerNameWithoutType); - if (type != null) { - Object valueTrans = typeTrans(value, type); - if (valueTrans == null) { - isFail.set(true); - ioTPrinter.printf( - "Line '%s', column '%s': '%s' can't convert to '%s'%n", - recordObj.getRecordNumber(), headerNameWithoutType, value, type); - } else { - values.add(valueTrans); - measurements.add(headerNameWithoutType); - types.add(type); - pointSize.getAndIncrement(); - } - } - } - } - if (Boolean.TRUE.equals(isFail.get())) { - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - } - if (!measurements.isEmpty()) { - times.add(parseTimestamp(recordObj.get(timeColumn))); - typesList.add(types); - valuesList.add(values); - measurementsList.add(measurements); - } - }); - if (!times.isEmpty()) { - writeAndEmptyDataSet(deviceName.get(), times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - if (!failedRecords.isEmpty()) { - writeFailedLinesFile(headerNames, failedFilePath, failedRecords); - } - ioTPrinter.println("Import completely!"); - } - - private static void writeFailedLinesFile( - List headerNames, String failedFilePath, ArrayList> failedRecords) { - int fileIndex = 0; - int from = 0; - int failedRecordsSize = failedRecords.size(); - int restFailedRecords = failedRecordsSize; - while (from < failedRecordsSize) { - int step = Math.min(restFailedRecords, linesPerFailedFile); - writeCsvFile( - headerNames, - failedRecords.subList(from, from + step), - failedFilePath + "_" + fileIndex++); - from += step; - restFailedRecords -= step; - } - } - - private static void writeAndEmptyDataSet( - String device, - List times, - List> typesList, - List> valuesList, - List> measurementsList, - int retryTime) { - try { - if (Boolean.FALSE.equals(aligned)) { - session.insertRecordsOfOneDevice(device, times, measurementsList, typesList, valuesList); - } else { - session.insertAlignedRecordsOfOneDevice( - device, times, measurementsList, typesList, valuesList); - } - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - try { - session.open(); - } catch (IoTDBConnectionException ex) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - } - writeAndEmptyDataSet(device, times, typesList, valuesList, measurementsList, --retryTime); - } - } catch (StatementExecutionException e) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - System.exit(1); - } finally { - times.clear(); - typesList.clear(); - valuesList.clear(); - measurementsList.clear(); - } - } - - private static void writeAndEmptyDataSet( - List deviceIds, - List times, - List> typesList, - List> valuesList, - List> measurementsList, - int retryTime) { - try { - if (Boolean.FALSE.equals(aligned)) { - session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); - } else { - session.insertAlignedRecords(deviceIds, times, measurementsList, typesList, valuesList); - } - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - try { - session.open(); - } catch (IoTDBConnectionException ex) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - } - writeAndEmptyDataSet( - deviceIds, times, typesList, valuesList, measurementsList, --retryTime); - } - } catch (StatementExecutionException e) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - System.exit(1); - } finally { - deviceIds.clear(); - times.clear(); - typesList.clear(); - valuesList.clear(); - measurementsList.clear(); - } - } - - /** - * read data from the CSV file - * - * @param path - * @return CSVParser csv parser - * @throws IOException when reading the csv file failed. - */ - private static CSVParser readCsvFile(String path) throws IOException { - return CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setQuote('`') - .setEscape('\\') - .setIgnoreEmptyLines(true) - .build() - .parse(new InputStreamReader(new FileInputStream(path))); - } - - /** - * parse deviceNames, measurementNames(aligned by time), headerType from headers - * - * @param headerNames - * @param deviceAndMeasurementNames - * @param headerTypeMap - * @param headerNameMap - */ - @SuppressWarnings( - "squid:S135") // ignore for loops should not contain more than a single "break" or "continue" - // statement - private static void parseHeaders( - List headerNames, - @Nullable HashMap> deviceAndMeasurementNames, - HashMap headerTypeMap, - HashMap headerNameMap) - throws IllegalPathException { - String regex = "(?<=\\()\\S+(?=\\))"; - Pattern pattern = Pattern.compile(regex); - for (String headerName : headerNames) { - if ("Time".equalsIgnoreCase(filterBomHeader(headerName))) { - timeColumn = headerName; - continue; - } else if ("Device".equalsIgnoreCase(headerName)) { - deviceColumn = headerName; - continue; - } - Matcher matcher = pattern.matcher(headerName); - String type; - String headerNameWithoutType; - if (matcher.find()) { - type = matcher.group(); - headerNameWithoutType = headerName.replace("(" + type + ")", "").replaceAll("\\s+", ""); - headerNameMap.put(headerNameWithoutType, headerName); - headerTypeMap.put(headerNameWithoutType, getType(type)); - } else { - headerNameWithoutType = headerName; - headerNameMap.put(headerName, headerName); - } - String[] split = PathUtils.splitPathToDetachedNodes(headerNameWithoutType); - String measurementName = split[split.length - 1]; - String deviceName = StringUtils.join(Arrays.copyOfRange(split, 0, split.length - 1), '.'); - if (deviceAndMeasurementNames != null) { - deviceAndMeasurementNames.putIfAbsent(deviceName, new ArrayList<>()); - deviceAndMeasurementNames.get(deviceName).add(measurementName); - } - } - } - - /** - * query data type of timeseries from IoTDB - * - * @param deviceNames - * @param headerTypeMap - * @param alignedType - * @throws IoTDBConnectionException - * @throws StatementExecutionException - */ - private static void queryType( - Set deviceNames, HashMap headerTypeMap, String alignedType) { - for (String deviceName : deviceNames) { - String sql = "show timeseries " + deviceName + ".*"; - SessionDataSet sessionDataSet = null; - try { - sessionDataSet = session.executeQueryStatement(sql); - int tsIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); - int dtIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); - while (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - String timeseries = fields.get(tsIndex).getStringValue(); - String dataType = fields.get(dtIndex).getStringValue(); - if (Objects.equals(alignedType, "Time")) { - headerTypeMap.put(timeseries, getType(dataType)); - } else if (Objects.equals(alignedType, deviceColumn)) { - String[] split = PathUtils.splitPathToDetachedNodes(timeseries); - String measurement = split[split.length - 1]; - headerTypeMap.put(measurement, getType(dataType)); - } - } - } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { - ioTPrinter.println( - "Meet error when query the type of timeseries because " + e.getMessage()); - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - System.exit(1); - } - } - } - - /** - * return the TSDataType - * - * @param typeStr - * @return - */ - private static TSDataType getType(String typeStr) { - try { - return TSDataType.valueOf(typeStr); - } catch (Exception e) { - return null; - } - } - - /** - * if data type of timeseries is not defined in headers of schema, this method will be called to - * do type inference - * - * @param strValue - * @return - */ - private static TSDataType typeInfer(String strValue) { - if (strValue.contains("\"")) { - return strValue.length() <= 512 + 2 ? STRING : TEXT; - } - if (isBoolean(strValue)) { - return TYPE_INFER_KEY_DICT.get(DATATYPE_BOOLEAN); - } else if (isNumber(strValue)) { - if (!strValue.contains(TsFileConstant.PATH_SEPARATOR)) { - if (isConvertFloatPrecisionLack(StringUtils.trim(strValue))) { - return TYPE_INFER_KEY_DICT.get(DATATYPE_LONG); - } - return TYPE_INFER_KEY_DICT.get(DATATYPE_INT); - } else { - return TYPE_INFER_KEY_DICT.get(DATATYPE_FLOAT); - } - } else if (DATATYPE_NULL.equals(strValue) || DATATYPE_NULL.toUpperCase().equals(strValue)) { - return null; - // "NaN" is returned if the NaN Literal is given in Parser - } else if (DATATYPE_NAN.equals(strValue)) { - return TYPE_INFER_KEY_DICT.get(DATATYPE_NAN); - } else if (strValue.length() <= 512) { - return STRING; - } else { - return TEXT; - } - } - - static boolean isNumber(String s) { - if (s == null || s.equals(DATATYPE_NAN)) { - return false; - } - try { - Double.parseDouble(s); - } catch (NumberFormatException e) { - return false; - } - return true; - } - - private static boolean isBoolean(String s) { - return s.equalsIgnoreCase(SqlConstant.BOOLEAN_TRUE) - || s.equalsIgnoreCase(SqlConstant.BOOLEAN_FALSE); - } - - private static boolean isConvertFloatPrecisionLack(String s) { - return Long.parseLong(s) > (2 << 24); - } - - /** - * @param value - * @param type - * @return - */ - private static Object typeTrans(String value, TSDataType type) { - try { - switch (type) { - case TEXT: - case STRING: - if (value.startsWith("\"") && value.endsWith("\"")) { - return value.substring(1, value.length() - 1); - } - return value; - case BOOLEAN: - if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { - return null; - } - return Boolean.parseBoolean(value); - case INT32: - return Integer.parseInt(value); - case INT64: - return Long.parseLong(value); - case FLOAT: - return Float.parseFloat(value); - case DOUBLE: - return Double.parseDouble(value); - default: - return null; - } - } catch (NumberFormatException e) { - return null; - } - } - - private static long parseTimestamp(String str) { - long timestamp; - try { - timestamp = Long.parseLong(str); - } catch (NumberFormatException e) { - timestamp = DateTimeUtils.convertDatetimeStrToLong(str, zoneId, timestampPrecision); - } - return timestamp; - } - - private static String filterBomHeader(String s) { - byte[] bom = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; - byte[] bytes = Arrays.copyOf(s.getBytes(), 3); - if (Arrays.equals(bom, bytes)) { - return s.substring(1); - } - return s; - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java deleted file mode 100644 index 3a7b8ab60e5d0..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java +++ /dev/null @@ -1,647 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.TSEncoding; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; - -/** Import Schema CSV file. */ -public class ImportSchema extends AbstractSchemaTool { - - private static final String FILE_ARGS = "s"; - private static final String FILE_NAME = "source"; - private static final String FILE_ARGS_NAME = "sourceDir/sourceFile"; - - private static final String FAILED_FILE_ARGS = "fd"; - private static final String FAILED_FILE_NAME = "fail_dir"; - private static final String FAILED_FILE_ARGS_NAME = "failDir"; - - private static final String ALIGNED_ARGS = "aligned"; - private static Boolean aligned = false; - - private static final String BATCH_POINT_SIZE_ARGS = "batch"; - private static final String BATCH_POINT_SIZE_NAME = "batch_size"; - private static final String BATCH_POINT_SIZE_ARGS_NAME = "batchSize"; - private static int batchPointSize = 10_000; - - private static final String CSV_SUFFIXS = "csv"; - - private static final String LINES_PER_FAILED_FILE_ARGS = "lpf"; - private static final String LINES_PER_FAILED_FILE_NAME = "lines_per_file"; - private static final String LINES_PER_FAILED_FILE_ARGS_NAME = "linesPerFile"; - private static final String IMPORT_SCHEMA_CLI_PREFIX = "ImportSchema"; - private static int linesPerFailedFile = 10000; - - private static String targetPath; - private static String failedFileDirectory = null; - - private static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because "; - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - - /** - * create the commandline options. - * - * @return object Options - */ - private static Options createOptions() { - Options options = createNewOptions(); - - Option opFile = - Option.builder(FILE_ARGS) - .required() - .longOpt(FILE_NAME) - .hasArg() - .argName(FILE_ARGS_NAME) - .desc( - "If input a file path, load a csv file, " - + "otherwise load all csv file under this directory (required)") - .build(); - options.addOption(opFile); - - Option opFailedFile = - Option.builder(FAILED_FILE_ARGS) - .longOpt(FAILED_FILE_NAME) - .hasArg() - .argName(FAILED_FILE_ARGS_NAME) - .desc( - "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)") - .build(); - options.addOption(opFailedFile); - - Option opBatchPointSize = - Option.builder(BATCH_POINT_SIZE_ARGS) - .longOpt(BATCH_POINT_SIZE_NAME) - .hasArg() - .argName(BATCH_POINT_SIZE_ARGS_NAME) - .desc("10000 (only not aligned optional)") - .build(); - options.addOption(opBatchPointSize); - - Option opFailedLinesPerFile = - Option.builder(LINES_PER_FAILED_FILE_ARGS) - .longOpt(LINES_PER_FAILED_FILE_NAME) - .hasArg() - .argName(LINES_PER_FAILED_FILE_ARGS_NAME) - .desc("Lines per failed file") - .build(); - options.addOption(opFailedLinesPerFile); - - Option opHelp = - Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help information").build(); - options.addOption(opHelp); - return options; - } - - /** - * parse optional params - * - * @param commandLine - */ - private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { - targetPath = commandLine.getOptionValue(FILE_ARGS); - if (commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS) != null) { - batchPointSize = Integer.parseInt(commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS)); - } - if (commandLine.getOptionValue(FAILED_FILE_ARGS) != null) { - failedFileDirectory = commandLine.getOptionValue(FAILED_FILE_ARGS); - File file = new File(failedFileDirectory); - if (!file.isDirectory()) { - file.mkdir(); - failedFileDirectory = file.getAbsolutePath() + File.separator; - } else if (!failedFileDirectory.endsWith("/") && !failedFileDirectory.endsWith("\\")) { - failedFileDirectory += File.separator; - } - } - if (commandLine.getOptionValue(ALIGNED_ARGS) != null) { - aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS)); - } - if (commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS) != null) { - linesPerFailedFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS)); - } - } - - public static void main(String[] args) throws IoTDBConnectionException { - Options options = createOptions(); - HelpFormatter hf = new HelpFormatter(); - hf.setOptionComparator(null); - hf.setWidth(MAX_HELP_CONSOLE_WIDTH); - CommandLine commandLine = null; - CommandLineParser parser = new DefaultParser(); - - if (args == null || args.length == 0) { - ioTPrinter.println("Too few params input, please check the following hint."); - hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - try { - commandLine = parser.parse(options, args); - } catch (org.apache.commons.cli.ParseException e) { - ioTPrinter.println("Parse error: " + e.getMessage()); - hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - if (commandLine.hasOption(HELP_ARGS)) { - hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - try { - parseBasicParams(commandLine); - String filename = commandLine.getOptionValue(FILE_ARGS); - if (filename == null) { - hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - parseSpecialParams(commandLine); - } catch (ArgsErrorException e) { - ioTPrinter.println("Args error: " + e.getMessage()); - System.exit(CODE_ERROR); - } catch (Exception e) { - ioTPrinter.println("Encounter an error, because: " + e.getMessage()); - System.exit(CODE_ERROR); - } - System.exit(importFromTargetPath(host, Integer.parseInt(port), username, password, targetPath)); - } - - /** - * Specifying a CSV file or a directory including CSV files that you want to import. This method - * can be offered to console cli to implement importing CSV file by command. - * - * @param host - * @param port - * @param username - * @param password - * @param targetPath a CSV file or a directory including CSV files - * @return the status code - * @throws IoTDBConnectionException - */ - @SuppressWarnings({"squid:S2093"}) // ignore try-with-resources - public static int importFromTargetPath( - String host, int port, String username, String password, String targetPath) { - try { - session = new Session(host, port, username, password, false); - session.open(false); - File file = new File(targetPath); - if (file.isFile()) { - importFromSingleFile(file); - } else if (file.isDirectory()) { - File[] files = file.listFiles(); - if (files == null) { - return CODE_OK; - } - // 按文件名排序 - Arrays.sort(files, (f1, f2) -> f1.getName().compareTo(f2.getName())); - for (File subFile : files) { - if (subFile.isFile()) { - importFromSingleFile(subFile); - } - } - } else { - ioTPrinter.println("File not found!"); - return CODE_ERROR; - } - } catch (IoTDBConnectionException e) { - ioTPrinter.println("Encounter an error when connecting to server, because " + e.getMessage()); - return CODE_ERROR; - } finally { - if (session != null) { - try { - session.close(); - } catch (IoTDBConnectionException e) { - ; - } - } - } - return CODE_OK; - } - - /** - * import the CSV file and load headers and records. - * - * @param file the File object of the CSV file that you want to import. - */ - private static void importFromSingleFile(File file) { - if (file.getName().endsWith(CSV_SUFFIXS)) { - try { - CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); - List headerNames = csvRecords.getHeaderNames(); - Stream records = csvRecords.stream(); - if (headerNames.isEmpty()) { - ioTPrinter.println(file.getName() + " : Empty file!"); - return; - } - if (!checkHeader(headerNames)) { - return; - } - String failedFilePath = null; - if (failedFileDirectory == null) { - failedFilePath = file.getAbsolutePath() + ".failed"; - } else { - failedFilePath = failedFileDirectory + file.getName() + ".failed"; - } - writeScheme(file.getName(), headerNames, records, failedFilePath); - } catch (IOException | IllegalPathException e) { - ioTPrinter.println( - file.getName() + " : CSV file read exception because: " + e.getMessage()); - } - } else { - ioTPrinter.println(file.getName() + " : The file name must end with \"csv\"!"); - } - } - - /** - * if the data is aligned by time, the data will be written by this method. - * - * @param headerNames the header names of CSV file - * @param records the records of CSV file - * @param failedFilePath the directory to save the failed files - */ - @SuppressWarnings("squid:S3776") - private static void writeScheme( - String fileName, List headerNames, Stream records, String failedFilePath) - throws IllegalPathException { - List paths = new ArrayList<>(); - List dataTypes = new ArrayList<>(); - List encodings = new ArrayList<>(); - List compressors = new ArrayList<>(); - List pathsWithAlias = new ArrayList<>(); - List dataTypesWithAlias = new ArrayList<>(); - List encodingsWithAlias = new ArrayList<>(); - List compressorsWithAlias = new ArrayList<>(); - List measurementAlias = new ArrayList<>(); - - AtomicReference hasStarted = new AtomicReference<>(false); - AtomicInteger pointSize = new AtomicInteger(0); - ArrayList> failedRecords = new ArrayList<>(); - records.forEach( - recordObj -> { - boolean failed = false; - if (!aligned) { - if (Boolean.FALSE.equals(hasStarted.get())) { - hasStarted.set(true); - } else if (pointSize.get() >= batchPointSize) { - try { - if (CollectionUtils.isNotEmpty(paths)) { - writeAndEmptyDataSet( - paths, dataTypes, encodings, compressors, null, null, null, null, 3); - } - } catch (Exception e) { - paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); - } - try { - if (CollectionUtils.isNotEmpty(pathsWithAlias)) { - writeAndEmptyDataSet( - pathsWithAlias, - dataTypesWithAlias, - encodingsWithAlias, - compressorsWithAlias, - null, - null, - null, - measurementAlias, - 3); - } - } catch (Exception e) { - paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); - } - paths.clear(); - dataTypes.clear(); - encodings.clear(); - compressors.clear(); - measurementAlias.clear(); - pointSize.set(0); - } - } else { - paths.clear(); - dataTypes.clear(); - encodings.clear(); - compressors.clear(); - measurementAlias.clear(); - } - String path = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(0))); - String alias = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(1))); - String dataTypeRaw = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(2))); - TSDataType dataType = typeInfer(dataTypeRaw); - String encodingTypeRaw = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(3))); - TSEncoding encodingType = encodingInfer(encodingTypeRaw); - String compressionTypeRaw = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(4))); - CompressionType compressionType = compressInfer(compressionTypeRaw); - if (StringUtils.isBlank(path) || path.trim().startsWith(SYSTEM_DATABASE)) { - ioTPrinter.println( - String.format( - "Line '%s', column '%s': illegal path %s", - recordObj.getRecordNumber(), headerNames, path)); - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - failed = true; - } else if (ObjectUtils.isEmpty(dataType)) { - ioTPrinter.println( - String.format( - "Line '%s', column '%s': '%s' unknown dataType %n", - recordObj.getRecordNumber(), path, dataTypeRaw)); - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - failed = true; - } else if (ObjectUtils.isEmpty(encodingType)) { - ioTPrinter.println( - String.format( - "Line '%s', column '%s': '%s' unknown encodingType %n", - recordObj.getRecordNumber(), path, encodingTypeRaw)); - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - failed = true; - } else if (ObjectUtils.isEmpty(compressionType)) { - ioTPrinter.println( - String.format( - "Line '%s', column '%s': '%s' unknown compressionType %n", - recordObj.getRecordNumber(), path, compressionTypeRaw)); - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - failed = true; - } else { - if (StringUtils.isBlank(alias)) { - paths.add(path); - dataTypes.add(dataType); - encodings.add(encodingType); - compressors.add(compressionType); - } else { - pathsWithAlias.add(path); - dataTypesWithAlias.add(dataType); - encodingsWithAlias.add(encodingType); - compressorsWithAlias.add(compressionType); - measurementAlias.add(alias); - } - pointSize.getAndIncrement(); - } - if (!failed && aligned) { - String deviceId = path.substring(0, path.lastIndexOf(".")); - paths.add(0, path.substring(deviceId.length() + 1)); - writeAndEmptyDataSetAligned( - deviceId, paths, dataTypes, encodings, compressors, measurementAlias, 3); - } - }); - try { - if (CollectionUtils.isNotEmpty(paths)) { - writeAndEmptyDataSet(paths, dataTypes, encodings, compressors, null, null, null, null, 3); - } - } catch (Exception e) { - paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); - } - try { - if (CollectionUtils.isNotEmpty(pathsWithAlias)) { - writeAndEmptyDataSet( - pathsWithAlias, - dataTypesWithAlias, - encodingsWithAlias, - compressorsWithAlias, - null, - null, - null, - measurementAlias, - 3); - } - } catch (Exception e) { - pathsWithAlias.forEach(t -> failedRecords.add(Collections.singletonList(t))); - } - pointSize.set(0); - if (!failedRecords.isEmpty()) { - writeFailedLinesFile(failedFilePath, failedRecords); - } - if (Boolean.TRUE.equals(hasStarted.get())) { - if (!failedRecords.isEmpty()) { - ioTPrinter.println(fileName + " : Import completely fail!"); - } else { - ioTPrinter.println(fileName + " : Import completely successful!"); - } - } else { - ioTPrinter.println(fileName + " : No records!"); - } - } - - private static boolean checkHeader(List headerNames) { - if (CollectionUtils.isNotEmpty(headerNames) - && new HashSet<>(headerNames).size() == HEAD_COLUMNS.size()) { - List strangers = - headerNames.stream().filter(t -> !HEAD_COLUMNS.contains(t)).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(strangers)) { - ioTPrinter.println( - "The header of the CSV file to be imported is illegal. The correct format is \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"); - return false; - } - } - return true; - } - - private static void writeFailedLinesFile( - String failedFilePath, ArrayList> failedRecords) { - int fileIndex = 0; - int from = 0; - int failedRecordsSize = failedRecords.size(); - int restFailedRecords = failedRecordsSize; - while (from < failedRecordsSize) { - int step = Math.min(restFailedRecords, linesPerFailedFile); - writeCsvFile(failedRecords.subList(from, from + step), failedFilePath + "_" + fileIndex++); - from += step; - restFailedRecords -= step; - } - } - - private static void writeAndEmptyDataSet( - List paths, - List dataTypes, - List encodings, - List compressors, - List> propsList, - List> tagsList, - List> attributesList, - List measurementAliasList, - int retryTime) - throws StatementExecutionException { - try { - session.createMultiTimeseries( - paths, - dataTypes, - encodings, - compressors, - propsList, - tagsList, - attributesList, - measurementAliasList); - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - try { - session.open(); - } catch (IoTDBConnectionException ex) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - } - writeAndEmptyDataSet( - paths, - dataTypes, - encodings, - compressors, - propsList, - tagsList, - attributesList, - measurementAliasList, - --retryTime); - } - } catch (StatementExecutionException e) { - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - throw e; - } - } - - private static void writeAndEmptyDataSetAligned( - String deviceId, - List measurements, - List dataTypes, - List encodings, - List compressors, - List measurementAliasList, - int retryTime) { - try { - session.createAlignedTimeseries( - deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - try { - session.open(); - } catch (IoTDBConnectionException ex) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - } - writeAndEmptyDataSetAligned( - deviceId, - measurements, - dataTypes, - encodings, - compressors, - measurementAliasList, - --retryTime); - } - } catch (StatementExecutionException e) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - System.exit(1); - } finally { - deviceId = null; - measurements.clear(); - dataTypes.clear(); - encodings.clear(); - compressors.clear(); - measurementAliasList.clear(); - } - } - - /** - * read data from the CSV file - * - * @param path - * @return CSVParser csv parser - * @throws IOException when reading the csv file failed. - */ - private static CSVParser readCsvFile(String path) throws IOException { - return CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setQuote('`') - .setEscape('\\') - .setIgnoreEmptyLines(true) - .build() - .parse(new InputStreamReader(new FileInputStream(path))); - } - - /** - * @param typeStr - * @return - */ - private static TSDataType typeInfer(String typeStr) { - try { - if (StringUtils.isNotBlank(typeStr)) { - return TSDataType.valueOf(typeStr); - } - } catch (IllegalArgumentException e) { - ; - } - return null; - } - - private static CompressionType compressInfer(String compressionType) { - try { - if (StringUtils.isNotBlank(compressionType)) { - return CompressionType.valueOf(compressionType); - } - } catch (IllegalArgumentException e) { - ; - } - return null; - } - - private static TSEncoding encodingInfer(String encodingType) { - try { - if (StringUtils.isNotBlank(encodingType)) { - return TSEncoding.valueOf(encodingType); - } - } catch (IllegalArgumentException e) { - ; - } - return null; - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java deleted file mode 100644 index 576d2927beb3f..0000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.session.pool.SessionPool; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.ParseException; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.LongAdder; - -public class ImportTsFile extends AbstractTsFileTool { - - private static final String SOURCE_ARGS = "s"; - private static final String SOURCE_NAME = "source"; - - private static final String ON_SUCCESS_ARGS = "os"; - private static final String ON_SUCCESS_NAME = "on_success"; - - private static final String SUCCESS_DIR_ARGS = "sd"; - private static final String SUCCESS_DIR_NAME = "success_dir"; - - private static final String FAIL_DIR_ARGS = "fd"; - private static final String FAIL_DIR_NAME = "fail_dir"; - - private static final String ON_FAIL_ARGS = "of"; - private static final String ON_FAIL_NAME = "on_fail"; - - private static final String THREAD_NUM_ARGS = "tn"; - private static final String THREAD_NUM_NAME = "thread_num"; - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - - private static final String TS_FILE_CLI_PREFIX = "ImportTsFile"; - - private static final String RESOURCE = ".resource"; - private static final String MODS = ".mods"; - - private static String source; - private static String sourceFullPath; - - private static String successDir = "success/"; - private static String failDir = "fail/"; - - private static Operation successOperation; - private static Operation failOperation; - - private static int threadNum = 8; - - private static final LongAdder loadFileSuccessfulNum = new LongAdder(); - private static final LongAdder loadFileFailedNum = new LongAdder(); - private static final LongAdder processingLoadSuccessfulFileSuccessfulNum = new LongAdder(); - private static final LongAdder processingLoadFailedFileSuccessfulNum = new LongAdder(); - - private static final LinkedBlockingQueue tsfileQueue = new LinkedBlockingQueue<>(); - private static final Set tsfileSet = new HashSet<>(); - private static final Set resourceOrModsSet = new HashSet<>(); - - private static SessionPool sessionPool; - - private static void createOptions() { - createBaseOptions(); - - Option opSource = - Option.builder(SOURCE_ARGS) - .longOpt(SOURCE_NAME) - .argName(SOURCE_NAME) - .required() - .hasArg() - .desc( - "The source file or directory path, " - + "which can be a tsfile or a directory containing tsfiles. (required)") - .build(); - options.addOption(opSource); - - Option opOnSuccess = - Option.builder(ON_SUCCESS_ARGS) - .longOpt(ON_SUCCESS_NAME) - .argName(ON_SUCCESS_NAME) - .required() - .hasArg() - .desc( - "When loading tsfile successfully, do operation on tsfile (and its .resource and .mods files), " - + "optional parameters are none, mv, cp, delete. (required)") - .build(); - options.addOption(opOnSuccess); - - Option opOnFail = - Option.builder(ON_FAIL_ARGS) - .longOpt(ON_FAIL_NAME) - .argName(ON_FAIL_NAME) - .required() - .hasArg() - .desc( - "When loading tsfile fail, do operation on tsfile (and its .resource and .mods files), " - + "optional parameters are none, mv, cp, delete. (required)") - .build(); - options.addOption(opOnFail); - - Option opSuccessDir = - Option.builder(SUCCESS_DIR_ARGS) - .longOpt(SUCCESS_DIR_NAME) - .argName(SUCCESS_DIR_NAME) - .hasArg() - .desc("The target folder when 'os' is 'mv' or 'cp'.") - .build(); - options.addOption(opSuccessDir); - - Option opFailDir = - Option.builder(FAIL_DIR_ARGS) - .longOpt(FAIL_DIR_NAME) - .argName(FAIL_DIR_NAME) - .hasArg() - .desc("The target folder when 'of' is 'mv' or 'cp'.") - .build(); - options.addOption(opFailDir); - - Option opThreadNum = - Option.builder(THREAD_NUM_ARGS) - .longOpt(THREAD_NUM_NAME) - .argName(THREAD_NUM_NAME) - .hasArgs() - .desc("The number of threads used to import tsfile, default is 8.") - .build(); - options.addOption(opThreadNum); - } - - public static void main(String[] args) { - long startTime = System.currentTimeMillis(); - createOptions(); - - final CommandLineParser parser = new DefaultParser(); - - final HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setOptionComparator(null); - helpFormatter.setWidth(MAX_HELP_CONSOLE_WIDTH); - - if (args == null || args.length == 0) { - ioTPrinter.println("Too few arguments, please check the following hint."); - helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - - try { - if (parser.parse(helpOptions, args, true).hasOption(HELP_ARGS)) { - helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); - System.exit(CODE_OK); - } - } catch (ParseException e) { - ioTPrinter.println("Failed to parse the provided options: " + e.getMessage()); - helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - - CommandLine commandLine = null; - try { - commandLine = parser.parse(options, args, true); - } catch (ParseException e) { - ioTPrinter.println("Failed to parse the provided options: " + e.getMessage()); - helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); - System.exit(CODE_ERROR); - } - - try { - parseBasicParams(commandLine); - parseSpecialParams(commandLine); - } catch (Exception e) { - ioTPrinter.println("Encounter an error when parsing the provided options: " + e.getMessage()); - System.exit(CODE_ERROR); - } - - final int resultCode = importFromTargetPath(); - ioTPrinter.println( - "Successfully load " - + loadFileSuccessfulNum.sum() - + " tsfile(s) (--on_success operation(s): " - + processingLoadSuccessfulFileSuccessfulNum.sum() - + " succeed, " - + (loadFileSuccessfulNum.sum() - processingLoadSuccessfulFileSuccessfulNum.sum()) - + " failed)"); - ioTPrinter.println( - "Failed to load " - + loadFileFailedNum.sum() - + " file(s) (--on_fail operation(s): " - + processingLoadFailedFileSuccessfulNum.sum() - + " succeed, " - + (loadFileFailedNum.sum() - processingLoadFailedFileSuccessfulNum.sum()) - + " failed)"); - ioTPrinter.println("For more details, please check the log."); - ioTPrinter.println( - "Total operation time: " + (System.currentTimeMillis() - startTime) + " ms."); - ioTPrinter.println("Work has been completed!"); - System.exit(resultCode); - } - - private static void parseSpecialParams(CommandLine commandLine) { - source = commandLine.getOptionValue(SOURCE_ARGS); - if (!Files.exists(Paths.get(source))) { - ioTPrinter.println(String.format("Source file or directory %s does not exist", source)); - System.exit(CODE_ERROR); - } - - final String onSuccess = commandLine.getOptionValue(ON_SUCCESS_ARGS).trim().toLowerCase(); - final String onFail = commandLine.getOptionValue(ON_FAIL_ARGS).trim().toLowerCase(); - if (!Operation.isValidOperation(onSuccess) || !Operation.isValidOperation(onFail)) { - ioTPrinter.println("Args error: os/of must be one of none, mv, cp, delete"); - System.exit(CODE_ERROR); - } - - boolean isSuccessDirEqualsSourceDir = false; - if (Operation.MV.name().equalsIgnoreCase(onSuccess) - || Operation.CP.name().equalsIgnoreCase(onSuccess)) { - File dir = createSuccessDir(commandLine); - isSuccessDirEqualsSourceDir = isFileStoreEquals(source, dir); - } - - boolean isFailDirEqualsSourceDir = false; - if (Operation.MV.name().equalsIgnoreCase(onFail) - || Operation.CP.name().equalsIgnoreCase(onFail)) { - File dir = createFailDir(commandLine); - isFailDirEqualsSourceDir = isFileStoreEquals(source, dir); - } - - successOperation = Operation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); - failOperation = Operation.getOperation(onFail, isFailDirEqualsSourceDir); - - if (commandLine.getOptionValue(THREAD_NUM_ARGS) != null) { - threadNum = Integer.parseInt(commandLine.getOptionValue(THREAD_NUM_ARGS)); - } - } - - public static boolean isFileStoreEquals(String pathString, File dir) { - try { - return Objects.equals( - Files.getFileStore(Paths.get(pathString)), Files.getFileStore(dir.toPath())); - } catch (IOException e) { - ioTPrinter.println("IOException when checking file store: " + e.getMessage()); - return false; - } - } - - public static File createSuccessDir(CommandLine commandLine) { - if (commandLine.getOptionValue(SUCCESS_DIR_ARGS) != null) { - successDir = commandLine.getOptionValue(SUCCESS_DIR_ARGS); - } - File file = new File(successDir); - if (!file.isDirectory()) { - if (!file.mkdirs()) { - ioTPrinter.println(String.format("Failed to create %s %s", SUCCESS_DIR_NAME, successDir)); - System.exit(CODE_ERROR); - } - } - return file; - } - - public static File createFailDir(CommandLine commandLine) { - if (commandLine.getOptionValue(FAIL_DIR_ARGS) != null) { - failDir = commandLine.getOptionValue(FAIL_DIR_ARGS); - } - File file = new File(failDir); - if (!file.isDirectory()) { - if (!file.mkdirs()) { - ioTPrinter.println(String.format("Failed to create %s %s", FAIL_DIR_NAME, failDir)); - System.exit(CODE_ERROR); - } - } - return file; - } - - public static int importFromTargetPath() { - try { - final File file = new File(source); - sourceFullPath = file.getAbsolutePath(); - if (!file.isFile() && !file.isDirectory()) { - ioTPrinter.println(String.format("source file or directory %s does not exist", source)); - return CODE_ERROR; - } - - sessionPool = - new SessionPool.Builder() - .host(host) - .port(Integer.parseInt(port)) - .user(username) - .password(password) - .maxSize(threadNum + 1) - .enableCompression(false) - .enableRedirection(false) - .enableAutoFetch(false) - .build(); - sessionPool.setEnableQueryRedirection(false); - - traverseAndCollectFiles(file); - addNoResourceOrModsToQueue(); - ioTPrinter.println("Load file total number : " + tsfileQueue.size()); - asyncImportTsFiles(); - return CODE_OK; - } catch (InterruptedException e) { - ioTPrinter.println(String.format("Import tsfile fail: %s", e.getMessage())); - Thread.currentThread().interrupt(); - return CODE_ERROR; - } finally { - if (sessionPool != null) { - sessionPool.close(); - } - } - } - - public static void traverseAndCollectFiles(File file) throws InterruptedException { - if (file.isFile()) { - if (file.getName().endsWith(RESOURCE) || file.getName().endsWith(MODS)) { - resourceOrModsSet.add(file.getAbsolutePath()); - } else { - tsfileSet.add(file.getAbsolutePath()); - tsfileQueue.put(file.getAbsolutePath()); - } - } else if (file.isDirectory()) { - final File[] files = file.listFiles(); - if (files != null) { - for (File f : files) { - traverseAndCollectFiles(f); - } - } - } - } - - public static void addNoResourceOrModsToQueue() throws InterruptedException { - for (final String filePath : resourceOrModsSet) { - final String tsfilePath = - filePath.endsWith(RESOURCE) - ? filePath.substring(0, filePath.length() - RESOURCE.length()) - : filePath.substring(0, filePath.length() - MODS.length()); - if (!tsfileSet.contains(tsfilePath)) { - tsfileQueue.put(filePath); - } - } - } - - public static void asyncImportTsFiles() { - final List list = new ArrayList<>(threadNum); - for (int i = 0; i < threadNum; i++) { - final Thread thread = new Thread(ImportTsFile::importTsFile); - thread.start(); - list.add(thread); - } - list.forEach( - thread -> { - try { - thread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - ioTPrinter.println("importTsFile thread join interrupted: " + e.getMessage()); - } - }); - } - - public static void importTsFile() { - String filePath; - try { - while ((filePath = tsfileQueue.poll()) != null) { - final String sql = "load '" + filePath + "' onSuccess=none "; - - try { - sessionPool.executeNonQueryStatement(sql); - - loadFileSuccessfulNum.increment(); - ioTPrinter.println("Imported [ " + filePath + " ] file successfully!"); - - try { - processingFile(filePath, successDir, successOperation); - processingLoadSuccessfulFileSuccessfulNum.increment(); - ioTPrinter.println("Processed success file [ " + filePath + " ] successfully!"); - } catch (Exception processSuccessException) { - ioTPrinter.println( - "Failed to process success file [ " - + filePath - + " ]: " - + processSuccessException.getMessage()); - } - } catch (Exception e) { - // Reject because of memory controls, do retry later - if (Objects.nonNull(e.getMessage()) && e.getMessage().contains("memory")) { - ioTPrinter.println( - "Rejecting file [ " + filePath + " ] due to memory constraints, will retry later."); - tsfileQueue.put(filePath); - continue; - } - - loadFileFailedNum.increment(); - ioTPrinter.println("Failed to import [ " + filePath + " ] file: " + e.getMessage()); - - try { - processingFile(filePath, failDir, failOperation); - processingLoadFailedFileSuccessfulNum.increment(); - ioTPrinter.println("Processed fail file [ " + filePath + " ] successfully!"); - } catch (Exception processFailException) { - ioTPrinter.println( - "Failed to process fail file [ " - + filePath - + " ]: " - + processFailException.getMessage()); - } - } - } - } catch (InterruptedException e) { - ioTPrinter.println("Unexpected error occurred: " + e.getMessage()); - Thread.currentThread().interrupt(); - } catch (Exception e) { - ioTPrinter.println("Unexpected error occurred: " + e.getMessage()); - } - } - - public static void processingFile(String filePath, String dir, Operation operation) { - String relativePath = filePath.substring(sourceFullPath.length() + 1); - Path sourcePath = Paths.get(filePath); - - String target = dir + File.separator + relativePath.replace(File.separator, "_"); - Path targetPath = Paths.get(target); - - Path sourceResourcePath = Paths.get(sourcePath + RESOURCE); - sourceResourcePath = Files.exists(sourceResourcePath) ? sourceResourcePath : null; - Path targetResourcePath = Paths.get(target + RESOURCE); - - Path sourceModsPath = Paths.get(sourcePath + MODS); - sourceModsPath = Files.exists(sourceModsPath) ? sourceModsPath : null; - Path targetModsPath = Paths.get(target + MODS); - - switch (operation) { - case DELETE: - { - try { - Files.deleteIfExists(sourcePath); - if (null != sourceResourcePath) { - Files.deleteIfExists(sourceResourcePath); - } - if (null != sourceModsPath) { - Files.deleteIfExists(sourceModsPath); - } - } catch (Exception e) { - ioTPrinter.println(String.format("Failed to delete file: %s", e.getMessage())); - } - break; - } - case CP: - { - try { - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); - if (null != sourceResourcePath) { - Files.copy( - sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); - } - if (null != sourceModsPath) { - Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); - } - } catch (Exception e) { - ioTPrinter.println(String.format("Failed to copy file: %s", e.getMessage())); - } - break; - } - case HARDLINK: - { - try { - Files.createLink(targetPath, sourcePath); - } catch (FileAlreadyExistsException e) { - ioTPrinter.println("Hardlink already exists: " + e.getMessage()); - } catch (Exception e) { - try { - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception copyException) { - ioTPrinter.println( - String.format("Failed to copy file: %s", copyException.getMessage())); - } - } - - try { - if (null != sourceResourcePath) { - Files.copy( - sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); - } - if (null != sourceModsPath) { - Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); - } - } catch (Exception e) { - ioTPrinter.println( - String.format("Failed to copy resource or mods file: %s", e.getMessage())); - } - break; - } - case MV: - { - try { - Files.move(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); - if (null != sourceResourcePath) { - Files.move( - sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); - } - if (null != sourceModsPath) { - Files.move(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); - } - } catch (Exception e) { - ioTPrinter.println(String.format("Failed to move file: %s", e.getMessage())); - } - break; - } - default: - break; - } - } - - public enum Operation { - NONE, - MV, - HARDLINK, - CP, - DELETE, - ; - - public static boolean isValidOperation(String operation) { - return "none".equalsIgnoreCase(operation) - || "mv".equalsIgnoreCase(operation) - || "cp".equalsIgnoreCase(operation) - || "delete".equalsIgnoreCase(operation); - } - - public static Operation getOperation(String operation, boolean isFileStoreEquals) { - switch (operation.toLowerCase()) { - case "none": - return Operation.NONE; - case "mv": - return Operation.MV; - case "cp": - if (isFileStoreEquals) { - return Operation.HARDLINK; - } else { - return Operation.CP; - } - case "delete": - return Operation.DELETE; - default: - ioTPrinter.println("Args error: os/of must be one of none, mv, cp, delete"); - System.exit(CODE_ERROR); - return null; - } - } - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/backup/IoTDBDataBackTool.java similarity index 99% rename from iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java rename to iotdb-client/cli/src/main/java/org/apache/iotdb/tool/backup/IoTDBDataBackTool.java index 7cc3556562b9d..62f27acbf622e 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/backup/IoTDBDataBackTool.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.iotdb.tool; +package org.apache.iotdb.tool.backup; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.tool.data.AbstractDataTool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java new file mode 100644 index 0000000000000..c93308289e62b --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.common; + +import org.apache.tsfile.enums.TSDataType; + +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Constants { + + // common + public static final int CODE_OK = 0; + public static final int CODE_ERROR = 1; + + public static final String HOST_ARGS = "h"; + public static final String HOST_NAME = "host"; + public static final String HOST_DESC = "Host Name (optional)"; + public static final String HOST_DEFAULT_VALUE = "127.0.0.1"; + + public static final String HELP_ARGS = "help"; + public static final String HELP_DESC = "Display help information"; + + public static final String PORT_ARGS = "p"; + public static final String PORT_NAME = "port"; + public static final String PORT_DESC = "Port (optional)"; + public static final String PORT_DEFAULT_VALUE = "6667"; + + public static final String PW_ARGS = "pw"; + public static final String PW_NAME = "password"; + public static final String PW_DESC = "Password (optional)"; + public static final String PW_DEFAULT_VALUE = "root"; + + public static final String USERNAME_ARGS = "u"; + public static final String USERNAME_NAME = "username"; + public static final String USERNAME_DESC = "Username (optional)"; + public static final String USERNAME_DEFAULT_VALUE = "root"; + + public static final String FILE_TYPE_ARGS = "ft"; + public static final String FILE_TYPE_NAME = "file_type"; + public static final String FILE_TYPE_ARGS_NAME = "format"; + public static final String FILE_TYPE_DESC = + "File type?You can choose tsfile)、csv) or sql).(required)"; + public static final String FILE_TYPE_DESC_EXPORT = + "Export file type ?You can choose tsfile)、csv) or sql).(required)"; + public static final String FILE_TYPE_DESC_IMPORT = + "Types of imported files: csv, sql, tsfile.(required)"; + + public static final String TIME_FORMAT_ARGS = "tf"; + public static final String TIME_FORMAT_NAME = "time_format"; + public static final String TIME_FORMAT_DESC = + "Output time Format in csv file. " + + "You can choose 1) timestamp, number, long 2) ISO8601, default 3) " + + "user-defined pattern like yyyy-MM-dd HH:mm:ss, default ISO8601.\n OutPut timestamp in sql file, No matter what time format is set(optional)"; + + public static final String TIME_ZONE_ARGS = "tz"; + public static final String TIME_ZONE_NAME = "timezone"; + public static final String TIME_ZONE_DESC = "Time Zone eg. +08:00 or -01:00 .(optional)"; + + public static final String TIMEOUT_ARGS = "timeout"; + public static final String TIMEOUT_NAME = "query_timeout"; + public static final String TIMEOUT_DESC = "Timeout for session query.(optional)"; + + public static final String ALIGNED_ARGS = "aligned"; + public static final String ALIGNED_NAME = "use_aligned"; + public static final String ALIGNED_ARGS_NAME_EXPORT = "export aligned insert sql"; + public static final String ALIGNED_ARGS_NAME_IMPORT = "use the aligned interface"; + public static final String ALIGNED_EXPORT_DESC = "Whether export to sql of aligned.(optional)"; + public static final String ALIGNED_IMPORT_DESC = + "Whether to use the interface of aligned.(optional)"; + + public static final String SQL_DIALECT_ARGS = "sql_dialect"; + public static final String SQL_DIALECT_DESC = + "Currently supports tree and table model, default tree. (optional)"; + public static final String SQL_DIALECT_VALUE_TREE = "tree"; + public static final String SQL_DIALECT_VALUE_TABLE = "table"; + + public static final String DB_ARGS = "db"; + public static final String DB_NAME = "database"; + public static final String DB_DESC = + "The database to be exported,only takes effect and required when sql_dialect is table .(optional)"; + + public static final String TABLE_ARGS = "table"; + public static final String TABLE_DESC = + "The table to be exported,only takes effect when sql_dialect is table.(optional)"; + public static final String TABLE_DESC_EXPORT = + TABLE_DESC + + ".If the '-q' parameter is specified, this parameter does not take effect. If the export type is tsfile or sql, this parameter is required. (optional)"; + public static final String TABLE_DESC_IMPORT = TABLE_DESC + " and file_type is csv. (optional)"; + + public static final String DATATYPE_BOOLEAN = "boolean"; + public static final String DATATYPE_INT = "int"; + public static final String DATATYPE_LONG = "long"; + public static final String DATATYPE_FLOAT = "float"; + public static final String DATATYPE_DOUBLE = "double"; + public static final String DATATYPE_TIMESTAMP = "timestamp"; + public static final String DATATYPE_DATE = "date"; + public static final String DATATYPE_BLOB = "blob"; + public static final String DATATYPE_NAN = "NaN"; + public static final String DATATYPE_TEXT = "text"; + public static final String DATATYPE_STRING = "string"; + public static final String DATATYPE_NULL = "null"; + public static final Map TYPE_INFER_KEY_DICT = new HashMap<>(); + + static { + TYPE_INFER_KEY_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); + TYPE_INFER_KEY_DICT.put(DATATYPE_INT, TSDataType.FLOAT); + TYPE_INFER_KEY_DICT.put(DATATYPE_LONG, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); + TYPE_INFER_KEY_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); + TYPE_INFER_KEY_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); + TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); + TYPE_INFER_KEY_DICT.put(DATATYPE_NAN, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_STRING, TSDataType.STRING); + } + + public static final Map TYPE_INFER_VALUE_DICT = new HashMap<>(); + + static { + TYPE_INFER_VALUE_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); + TYPE_INFER_VALUE_DICT.put(DATATYPE_INT, TSDataType.INT32); + TYPE_INFER_VALUE_DICT.put(DATATYPE_LONG, TSDataType.INT64); + TYPE_INFER_VALUE_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); + TYPE_INFER_VALUE_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); + TYPE_INFER_VALUE_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); + TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.DATE); + TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); + TYPE_INFER_VALUE_DICT.put(DATATYPE_TEXT, TSDataType.TEXT); + TYPE_INFER_VALUE_DICT.put(DATATYPE_STRING, TSDataType.STRING); + } + + public static final int MAX_HELP_CONSOLE_WIDTH = 92; + + public static final String CSV_SUFFIXS = "csv"; + public static final String TXT_SUFFIXS = "txt"; + public static final String SQL_SUFFIXS = "sql"; + public static final String TSFILE_SUFFIXS = "tsfile"; + + public static final String TSFILEDB_CLI_DIVIDE = "-------------------"; + public static final String COLON = ": "; + public static final String MINUS = "-"; + + public static final List HEAD_COLUMNS = + Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression"); + + // export constants + public static final String EXPORT_CLI_PREFIX = "Export Data"; + public static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema"; + + public static final String EXPORT_CLI_HEAD = + "Please obtain help information for the corresponding data type based on different parameters, for example:\n" + + "./export_data.sh -help tsfile\n" + + "./export_data.sh -help sql\n" + + "./export_data.sh -help csv"; + + public static final String SCHEMA_CLI_CHECK_IN_HEAD = + "Too few params input, please check the following hint."; + public static final String START_TIME_ARGS = "start_time"; + public static final String START_TIME_DESC = "The start time to be exported (optional)"; + + public static final String END_TIME_ARGS = "end_time"; + public static final String END_TIME_DESC = "The end time to be exported. (optional)"; + + public static final String TARGET_DIR_ARGS = "t"; + public static final String TARGET_DIR_NAME = "target"; + public static final String TARGET_DIR_ARGS_NAME = "target_directory"; + public static final String TARGET_DIR_DESC = "Target file directory (required)"; + public static final String TARGET_DIR_SUBSCRIPTION_DESC = + "Target file directory.default ./target (optional)"; + + public static final String TARGET_PATH_ARGS = "path"; + public static final String TARGET_PATH_ARGS_NAME = "path_pattern"; + public static final String TARGET_PATH_NAME = "exportPathPattern"; + public static final String TARGET_PATH_DESC = "Export Path Pattern (optional)"; + + public static final String QUERY_COMMAND_ARGS = "q"; + public static final String QUERY_COMMAND_NAME = "query"; + public static final String QUERY_COMMAND_ARGS_NAME = "query_command"; + public static final String QUERY_COMMAND_DESC = + "The query command that you want to execute.If sql_dialect is table The 'q' parameter is only applicable to export types of CSV, and is not available for other types.If the '- q' parameter is not empty, then the parameters' creatTime ',' EndTime 'and' table 'are not effective.(optional)"; + + public static final String TARGET_FILE_ARGS = "pfn"; + public static final String TARGET_FILE_NAME = "prefix_file_name"; + public static final String TARGET_FILE_DESC = "Export file name .(optional)"; + + public static final String DATA_TYPE_ARGS = "dt"; + public static final String DATA_TYPE_NAME = "datatype"; + public static final String DATA_TYPE_DESC = + "Will the data type of timeseries be printed in the head line of the CSV file?" + + '\n' + + "You can choose true) or false) . (optional)"; + + public static final String LINES_PER_FILE_ARGS = "lpf"; + public static final String LINES_PER_FILE_NAME = "lines_per_file"; + public static final String LINES_PER_FILE_DESC = + "Lines per dump file,only effective in tree model.(optional)"; + + public static final String DUMP_FILE_NAME_DEFAULT = "dump"; + + public static final String queryTableParamRequired = + "Either '-q' or '-table' is required when 'sql-dialect' is' table '"; + public static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because "; + public static final String INSERT_SQL_MEET_ERROR_MSG = "Meet error when insert sql because "; + public static final String COLUMN_SQL_MEET_ERROR_MSG = + "Meet error when get table columns information because "; + public static final String TARGET_DATABASE_NOT_EXIST_MSG = + "The target database %s does not exist"; + public static final String TARGET_TABLE_NOT_EXIST_MSG = + "There are no tables or the target table %s does not exist"; + public static final String TARGET_TABLE_EMPTY_MSG = + "There are no tables to export. Please check if the tables in the target database exist and if you have permission to access them."; + + public static final String[] TIME_FORMAT = + new String[] {"default", "long", "number", "timestamp"}; + + public static final String PATH_ARGS = "path"; + public static final String PATH_DESC = + "The path to be exported,only takes effect when sql_dialect is tree.(optional)"; + + public static final long memoryThreshold = 10 * 1024 * 1024; + + public static final String[] STRING_TIME_FORMAT = + new String[] { + "yyyy-MM-dd HH:mm:ss.SSSX", + "yyyy/MM/dd HH:mm:ss.SSSX", + "yyyy.MM.dd HH:mm:ss.SSSX", + "yyyy-MM-dd HH:mm:ssX", + "yyyy/MM/dd HH:mm:ssX", + "yyyy.MM.dd HH:mm:ssX", + "yyyy-MM-dd HH:mm:ss.SSSz", + "yyyy/MM/dd HH:mm:ss.SSSz", + "yyyy.MM.dd HH:mm:ss.SSSz", + "yyyy-MM-dd HH:mm:ssz", + "yyyy/MM/dd HH:mm:ssz", + "yyyy.MM.dd HH:mm:ssz", + "yyyy-MM-dd HH:mm:ss.SSS", + "yyyy/MM/dd HH:mm:ss.SSS", + "yyyy.MM.dd HH:mm:ss.SSS", + "yyyy-MM-dd HH:mm:ss", + "yyyy/MM/dd HH:mm:ss", + "yyyy.MM.dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss.SSSX", + "yyyy/MM/dd'T'HH:mm:ss.SSSX", + "yyyy.MM.dd'T'HH:mm:ss.SSSX", + "yyyy-MM-dd'T'HH:mm:ssX", + "yyyy/MM/dd'T'HH:mm:ssX", + "yyyy.MM.dd'T'HH:mm:ssX", + "yyyy-MM-dd'T'HH:mm:ss.SSSz", + "yyyy/MM/dd'T'HH:mm:ss.SSSz", + "yyyy.MM.dd'T'HH:mm:ss.SSSz", + "yyyy-MM-dd'T'HH:mm:ssz", + "yyyy/MM/dd'T'HH:mm:ssz", + "yyyy.MM.dd'T'HH:mm:ssz", + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy/MM/dd'T'HH:mm:ss.SSS", + "yyyy.MM.dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy/MM/dd'T'HH:mm:ss", + "yyyy.MM.dd'T'HH:mm:ss" + }; + + public static final String SUBSCRIPTION_CLI_PREFIX = "Export TsFile"; + public static final int MAX_RETRY_TIMES = 2; + public static final String LOOSE_RANGE = ""; + public static final boolean STRICT = false; + public static final String MODE = "snapshot"; + public static final boolean AUTO_COMMIT = true; + public static final String TABLE_MODEL = "table"; + public static final long AUTO_COMMIT_INTERVAL = 5000; + public static final long POLL_MESSAGE_TIMEOUT = 10000; + public static final String TOPIC_NAME_PREFIX = "topic_"; + public static final String GROUP_NAME_PREFIX = "group_"; + public static final String HANDLER = "TsFileHandler"; + public static final String CONSUMER_NAME_PREFIX = "consumer_"; + public static final SimpleDateFormat DATE_FORMAT_VIEW = new SimpleDateFormat("yyyyMMddHHmmssSSS"); + public static final String BASE_VIEW_TYPE = "BASE"; + public static final String HEADER_VIEW_TYPE = "ViewType"; + public static final String HEADER_TIMESERIES = "Timeseries"; + public static final String EXPORT_COMPLETELY = "Export completely!"; + public static final String EXPORT_SCHEMA_TABLES_SELECT = + "select * from information_schema.tables where database = '%s'"; + public static final String EXPORT_SCHEMA_TABLES_SHOW = "show tables details from %s"; + public static final String EXPORT_SCHEMA_TABLES_SHOW_DATABASES = "show databases"; + public static final String EXPORT_SCHEMA_COLUMNS_SELECT = + "select * from information_schema.columns where database like '%s' and table_name like '%s'"; + public static final String EXPORT_SCHEMA_COLUMNS_DESC = "desc %s.%s details"; + + // import constants + public static final String IMPORT_SCHEMA_CLI_PREFIX = "ImportSchema"; + public static final String IMPORT_CLI_PREFIX = "Import Data"; + + public static final String IMPORT_CLI_HEAD = + "Please obtain help information for the corresponding data type based on different parameters, for example:\n" + + "./import_data.sh -help tsfile\n" + + "./import_data.sh -help sql\n" + + "./import_data.sh -help csv"; + + public static final String FILE_ARGS = "s"; + public static final String FILE_NAME = "source"; + public static final String FILE_DESC = + "The local directory path of the script file (folder) to be loaded. (required)"; + + public static final String FAILED_FILE_ARGS = "fd"; + public static final String FAILED_FILE_NAME = "fail_dir"; + public static final String FAILED_FILE_ARGS_NAME = "failDir"; + public static final String FAILED_FILE_DESC = + "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)"; + + public static final String ON_SUCCESS_ARGS = "os"; + public static final String ON_SUCCESS_NAME = "on_success"; + public static final String ON_SUCCESS_DESC = + "When loading tsfile successfully, do operation on tsfile (and its .resource and .mods files), " + + "optional parameters are none, mv, cp, delete. (required)"; + + public static final String SUCCESS_DIR_ARGS = "sd"; + public static final String SUCCESS_DIR_NAME = "success_dir"; + public static final String SUCCESS_DIR_DESC = + "The target folder when 'os' is 'mv' or 'cp'.(optional)"; + + public static final String FAIL_DIR_ARGS = "fd"; + public static final String FAIL_DIR_NAME = "fail_dir"; + public static final String FAIL_DIR_CSV_DESC = + "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH.(optional)"; + public static final String FAIL_DIR_SQL_DESC = + "Specifying a directory to save failed file, default YOUR_SQL_FILE_PATH.(optional)"; + public static final String FAIL_DIR_TSFILE_DESC = + "The target folder when 'of' is 'mv' or 'cp'.(optional)"; + + public static final String ON_FAIL_ARGS = "of"; + public static final String ON_FAIL_NAME = "on_fail"; + public static final String ON_FAIL_DESC = + "When loading tsfile fail, do operation on tsfile (and its .resource and .mods files), " + + "optional parameters are none, mv, cp, delete. (required)"; + + public static final String THREAD_NUM_ARGS = "tn"; + public static final String THREAD_NUM_NAME = "thread_num"; + public static final String THREAD_NUM_DESC = + "The number of threads used to import tsfile, default is 8.(optional)"; + + public static final String BATCH_POINT_SIZE_ARGS = "batch"; + public static final String BATCH_POINT_SIZE_NAME = "batch_size"; + public static final String BATCH_POINT_SIZE_ARGS_NAME = "batch_size"; + public static final String BATCH_POINT_SIZE_DESC = "100000 (optional)"; + public static final String BATCH_POINT_SIZE_LIMIT_DESC = + "10000 (only not aligned and sql_dialect tree optional)"; + + public static final String TIMESTAMP_PRECISION_ARGS = "tp"; + public static final String TIMESTAMP_PRECISION_NAME = "timestamp_precision"; + public static final String TIMESTAMP_PRECISION_ARGS_NAME = "timestamp precision (ms/us/ns)"; + public static final String TIMESTAMP_PRECISION_DESC = "Timestamp precision (ms/us/ns).(optional)"; + + public static final String TYPE_INFER_ARGS = "ti"; + public static final String TYPE_INFER_NAME = "type_infer"; + public static final String TYPE_INFER_DESC = + "Define type info by option:\"boolean=text,int=long, ... (optional)"; + + public static final String LINES_PER_FAILED_FILE_ARGS = "lpf"; + public static final String LINES_PER_FAILED_FILE_ARGS_NAME = "lines_per_failed_file"; + public static final String LINES_PER_FAILED_FILE_DESC = + "Lines per failed file,only takes effect and required when sql_dialect is table .(option)"; + public static final String IMPORT_COMPLETELY = "Import completely!"; + public static final int BATCH_POINT_SIZE = 10000; +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java new file mode 100644 index 0000000000000..3769d855d161b --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.common; + +import org.apache.iotdb.cli.utils.IoTPrinter; + +public enum ImportTsFileOperation { + NONE, + MV, + HARDLINK, + CP, + DELETE, + ; + + public static boolean isValidOperation(String operation) { + return "none".equalsIgnoreCase(operation) + || "mv".equalsIgnoreCase(operation) + || "cp".equalsIgnoreCase(operation) + || "delete".equalsIgnoreCase(operation); + } + + public static ImportTsFileOperation getOperation(String operation, boolean isFileStoreEquals) { + switch (operation.toLowerCase()) { + case "none": + return ImportTsFileOperation.NONE; + case "mv": + return ImportTsFileOperation.MV; + case "cp": + if (isFileStoreEquals) { + return ImportTsFileOperation.HARDLINK; + } else { + return ImportTsFileOperation.CP; + } + case "delete": + return ImportTsFileOperation.DELETE; + default: + new IoTPrinter(System.out).println("Args error: os/of must be one of none, mv, cp, delete"); + System.exit(Constants.CODE_ERROR); + return null; + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java new file mode 100644 index 0000000000000..57ccff5c3e5c3 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java @@ -0,0 +1,1197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.common; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +public class OptionsUtil extends Constants { + + private static boolean isImport = true; + + public static void setIsImport(boolean isImport) { + OptionsUtil.isImport = isImport; + } + + public static Options createHelpOptions() { + final Options options = new Options(); + Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS).hasArg().desc(HELP_DESC).build(); + options.addOption(opHelp); + + Option opFileType = + Option.builder(FILE_TYPE_ARGS) + .longOpt(FILE_TYPE_NAME) + .argName(FILE_TYPE_ARGS_NAME) + .hasArg() + .desc(FILE_TYPE_DESC) + .build(); + options.addOption(opFileType); + + Option opSqlDialect = + Option.builder(SQL_DIALECT_ARGS) + .longOpt(SQL_DIALECT_ARGS) + .argName(SQL_DIALECT_ARGS) + .hasArg() + .desc(SQL_DIALECT_DESC) + .build(); + options.addOption(opSqlDialect); + + return options; + } + + public static Options createCommonOptions(Options options) { + Option opSqlDialect = + Option.builder(SQL_DIALECT_ARGS) + .longOpt(SQL_DIALECT_ARGS) + .argName(SQL_DIALECT_ARGS) + .hasArg() + .desc(SQL_DIALECT_DESC) + .build(); + options.addOption(opSqlDialect); + + Option opHost = + Option.builder(HOST_ARGS) + .longOpt(HOST_NAME) + .argName(HOST_NAME) + .hasArg() + .desc(HOST_DESC) + .build(); + options.addOption(opHost); + + Option opPort = + Option.builder(PORT_ARGS) + .longOpt(PORT_NAME) + .argName(PORT_NAME) + .hasArg() + .desc(PORT_DESC) + .build(); + options.addOption(opPort); + + Option opUsername = + Option.builder(USERNAME_ARGS) + .longOpt(USERNAME_NAME) + .argName(USERNAME_NAME) + .hasArg() + .desc(USERNAME_DESC) + .build(); + options.addOption(opUsername); + + Option opPassword = + Option.builder(PW_ARGS) + .longOpt(PW_NAME) + .optionalArg(true) + .argName(PW_NAME) + .hasArg() + .desc(PW_DESC) + .build(); + options.addOption(opPassword); + + return options; + } + + public static Options createImportCommonOptions() { + Options options = new Options(); + + Option opFileType = + Option.builder(FILE_TYPE_ARGS) + .longOpt(FILE_TYPE_NAME) + .argName(FILE_TYPE_ARGS_NAME) + .required() + .hasArg() + .desc(isImport ? FILE_TYPE_DESC_IMPORT : FILE_TYPE_DESC_EXPORT) + .build(); + options.addOption(opFileType); + + return createCommonOptions(options); + } + + public static Options createTreeImportCommonOptions() { + return createImportCommonOptions(); + } + + public static Options createTableImportCommonOptions() { + Options options = createImportCommonOptions(); + + Option opDatabase = + Option.builder(DB_ARGS) + .longOpt(DB_NAME) + .argName(DB_ARGS) + .hasArg() + .required() + .desc(DB_DESC) + .build(); + options.addOption(opDatabase); + + return options; + } + + public static Options createTreeExportCommonOptions() { + Options options = createImportCommonOptions(); + + Option opFile = + Option.builder(TARGET_DIR_ARGS) + .required() + .longOpt(TARGET_DIR_NAME) + .argName(TARGET_DIR_ARGS_NAME) + .hasArg() + .desc(TARGET_DIR_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(TARGET_FILE_ARGS) + .longOpt(TARGET_FILE_NAME) + .argName(TARGET_FILE_NAME) + .hasArg() + .desc(TARGET_FILE_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opQuery = + Option.builder(QUERY_COMMAND_ARGS) + .longOpt(QUERY_COMMAND_NAME) + .argName(QUERY_COMMAND_ARGS_NAME) + .hasArg() + .desc(QUERY_COMMAND_DESC) + .build(); + options.addOption(opQuery); + + Option opTimeOut = + Option.builder(TIMEOUT_ARGS) + .longOpt(TIMEOUT_NAME) + .argName(TIMEOUT_NAME) + .hasArg() + .desc(TIMEOUT_DESC) + .build(); + options.addOption(opTimeOut); + return options; + } + + public static Options createTableExportCommonOptions() { + final Options options = createImportCommonOptions(); + + Option opFile = + Option.builder(TARGET_DIR_ARGS) + .required() + .longOpt(TARGET_DIR_NAME) + .argName(TARGET_DIR_ARGS_NAME) + .hasArg() + .desc(TARGET_DIR_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(TARGET_FILE_ARGS) + .longOpt(TARGET_FILE_NAME) + .argName(TARGET_FILE_NAME) + .hasArg() + .desc(TARGET_FILE_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opQuery = + Option.builder(QUERY_COMMAND_ARGS) + .longOpt(QUERY_COMMAND_NAME) + .argName(QUERY_COMMAND_ARGS_NAME) + .hasArg() + .desc(QUERY_COMMAND_DESC) + .build(); + options.addOption(opQuery); + + Option opTimeOut = + Option.builder(TIMEOUT_ARGS) + .longOpt(TIMEOUT_NAME) + .argName(TIMEOUT_NAME) + .hasArg() + .desc(TIMEOUT_DESC) + .build(); + options.addOption(opTimeOut); + + Option opDatabase = + Option.builder(DB_ARGS) + .longOpt(DB_NAME) + .argName(DB_ARGS) + .hasArg() + .required() + .desc(DB_DESC) + .build(); + options.addOption(opDatabase); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opStartTime = + Option.builder(START_TIME_ARGS) + .longOpt(START_TIME_ARGS) + .argName(START_TIME_ARGS) + .hasArg() + .desc(START_TIME_DESC) + .build(); + options.addOption(opStartTime); + + Option opEndTime = + Option.builder(END_TIME_ARGS) + .longOpt(END_TIME_ARGS) + .argName(END_TIME_ARGS) + .hasArg() + .desc(END_TIME_DESC) + .build(); + options.addOption(opEndTime); + + return options; + } + + public static Options createTableExportCsvOptions() { + Options options = createTableExportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createTableExportTsFileOptions() { + Options options = createTableExportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createTableExportSqlOptions() { + Options options = createTableExportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createExportCsvOptions() { + Options options = createTreeExportCommonOptions(); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createExportSqlOptions() { + Options options = createTreeExportCommonOptions(); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_NAME) + .argName(ALIGNED_ARGS_NAME_EXPORT) + .hasArgs() + .desc(ALIGNED_EXPORT_DESC) + .build(); + options.addOption(opAligned); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createExportTsFileOptions() { + Options options = createTreeExportCommonOptions(); + return options; + } + + public static Options createImportCsvOptions() { + Options options = createTreeImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .required() + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_CSV_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .longOpt(LINES_PER_FAILED_FILE_ARGS_NAME) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_NAME) + .argName(ALIGNED_ARGS_NAME_IMPORT) + .hasArg() + .desc(ALIGNED_IMPORT_DESC) + .build(); + options.addOption(opAligned); + + Option opTypeInfer = + Option.builder(TYPE_INFER_ARGS) + .longOpt(TYPE_INFER_NAME) + .argName(TYPE_INFER_NAME) + .numberOfArgs(5) + .hasArgs() + .valueSeparator(',') + .desc(TYPE_INFER_DESC) + .build(); + options.addOption(opTypeInfer); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_ARGS_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createImportSqlOptions() { + Options options = createTreeImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_SQL_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArgs() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createImportTsFileOptions() { + Options options = createTreeImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(ON_SUCCESS_ARGS) + .longOpt(ON_SUCCESS_NAME) + .argName(ON_SUCCESS_NAME) + .required() + .hasArg() + .desc(ON_SUCCESS_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opSuccessDir = + Option.builder(SUCCESS_DIR_ARGS) + .longOpt(SUCCESS_DIR_NAME) + .argName(SUCCESS_DIR_NAME) + .hasArg() + .desc(SUCCESS_DIR_DESC) + .build(); + options.addOption(opSuccessDir); + + Option opOnFail = + Option.builder(ON_FAIL_ARGS) + .longOpt(ON_FAIL_NAME) + .argName(ON_FAIL_NAME) + .required() + .hasArg() + .desc(ON_FAIL_DESC) + .build(); + options.addOption(opOnFail); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_TSFILE_DESC) + .build(); + options.addOption(opFailDir); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + return options; + } + + public static Options createTableImportCsvOptions() { + Options options = createTableImportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_IMPORT) + .build(); + options.addOption(opTable); + + Option opFile = + Option.builder(FILE_ARGS) + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .required() + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_CSV_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .longOpt(LINES_PER_FAILED_FILE_ARGS_NAME) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_NAME) + .argName(ALIGNED_ARGS_NAME_IMPORT) + .hasArg() + .desc(ALIGNED_IMPORT_DESC) + .build(); + options.addOption(opAligned); + + Option opTypeInfer = + Option.builder(TYPE_INFER_ARGS) + .longOpt(TYPE_INFER_NAME) + .argName(TYPE_INFER_NAME) + .numberOfArgs(5) + .hasArgs() + .valueSeparator(',') + .desc(TYPE_INFER_DESC) + .build(); + options.addOption(opTypeInfer); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_ARGS_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createTableImportSqlOptions() { + Options options = createTableImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_SQL_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArgs() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createTableImportTsFileOptions() { + Options options = createTableImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(ON_SUCCESS_ARGS) + .longOpt(ON_SUCCESS_NAME) + .argName(ON_SUCCESS_NAME) + .required() + .hasArg() + .desc(ON_SUCCESS_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opSuccessDir = + Option.builder(SUCCESS_DIR_ARGS) + .longOpt(SUCCESS_DIR_NAME) + .argName(SUCCESS_DIR_NAME) + .hasArg() + .desc(SUCCESS_DIR_DESC) + .build(); + options.addOption(opSuccessDir); + + Option opOnFail = + Option.builder(ON_FAIL_ARGS) + .longOpt(ON_FAIL_NAME) + .argName(ON_FAIL_NAME) + .required() + .hasArg() + .desc(ON_FAIL_DESC) + .build(); + options.addOption(opOnFail); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_TSFILE_DESC) + .build(); + options.addOption(opFailDir); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + return options; + } + + public static Options createSubscriptionTsFileOptions() { + Options options = new Options(); + Option opSqlDialect = + Option.builder(SQL_DIALECT_ARGS) + .longOpt(SQL_DIALECT_ARGS) + .argName(SQL_DIALECT_ARGS) + .hasArg() + .desc(SQL_DIALECT_DESC) + .build(); + options.addOption(opSqlDialect); + + Option opHost = + Option.builder(HOST_ARGS) + .longOpt(HOST_NAME) + .argName(HOST_NAME) + .hasArg() + .desc(HOST_DESC) + .build(); + options.addOption(opHost); + + Option opPort = + Option.builder(PORT_ARGS) + .longOpt(PORT_NAME) + .argName(PORT_NAME) + .hasArg() + .desc(PORT_DESC) + .build(); + options.addOption(opPort); + + Option opUsername = + Option.builder(USERNAME_ARGS) + .longOpt(USERNAME_NAME) + .argName(USERNAME_NAME) + .hasArg() + .desc(USERNAME_DESC) + .build(); + options.addOption(opUsername); + + Option opPassword = + Option.builder(PW_ARGS) + .longOpt(PW_NAME) + .optionalArg(true) + .argName(PW_NAME) + .hasArg() + .desc(PW_DESC) + .build(); + options.addOption(opPassword); + + Option opPath = + Option.builder(PATH_ARGS) + .longOpt(PATH_ARGS) + .argName(PATH_ARGS) + .hasArg() + .desc(PATH_DESC) + .build(); + options.addOption(opPath); + + Option opDatabase = + Option.builder(DB_ARGS).longOpt(DB_NAME).argName(DB_ARGS).hasArg().desc(DB_DESC).build(); + options.addOption(opDatabase); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC) + .build(); + options.addOption(opTable); + Option opStartTime = + Option.builder(START_TIME_ARGS) + .longOpt(START_TIME_ARGS) + .argName(START_TIME_ARGS) + .hasArg() + .desc(START_TIME_DESC) + .build(); + options.addOption(opStartTime); + + Option opEndTime = + Option.builder(END_TIME_ARGS) + .longOpt(END_TIME_ARGS) + .argName(END_TIME_ARGS) + .hasArg() + .desc(END_TIME_DESC) + .build(); + options.addOption(opEndTime); + + Option opFile = + Option.builder(TARGET_DIR_ARGS) + .longOpt(TARGET_DIR_NAME) + .argName(TARGET_DIR_ARGS_NAME) + .hasArg() + .desc(TARGET_DIR_SUBSCRIPTION_DESC) + .build(); + options.addOption(opFile); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + + Option opHelp = + Option.builder(HELP_ARGS).longOpt(HELP_ARGS).hasArg(false).desc(HELP_DESC).build(); + options.addOption(opHelp); + return options; + } + + public static Options createExportSchemaOptions() { + Options options = createCommonOptions(new Options()); + Option opTargetFile = + Option.builder(TARGET_DIR_ARGS) + .required() + .longOpt(TARGET_DIR_ARGS_NAME) + .hasArg() + .argName(TARGET_DIR_NAME) + .desc(TARGET_DIR_DESC) + .build(); + options.addOption(opTargetFile); + + Option targetPathPattern = + Option.builder(TARGET_PATH_ARGS) + .longOpt(TARGET_PATH_ARGS_NAME) + .hasArg() + .argName(TARGET_PATH_NAME) + .desc(TARGET_PATH_DESC) + .build(); + options.addOption(targetPathPattern); + + Option targetFileName = + Option.builder(TARGET_FILE_ARGS) + .longOpt(TARGET_FILE_NAME) + .hasArg() + .argName(TARGET_FILE_NAME) + .desc(TARGET_FILE_DESC) + .build(); + options.addOption(targetFileName); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .hasArg() + .argName(LINES_PER_FILE_NAME) + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeout = + Option.builder(TIMEOUT_ARGS) + .longOpt(TIMEOUT_NAME) + .hasArg() + .argName(TIMEOUT_ARGS) + .desc(TIMEOUT_DESC) + .build(); + options.addOption(opTimeout); + + Option opDatabase = + Option.builder(DB_ARGS).longOpt(DB_NAME).argName(DB_ARGS).hasArg().desc(DB_DESC).build(); + options.addOption(opDatabase); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc(HELP_DESC).build(); + options.addOption(opHelp); + + return options; + } + + public static Options createImportSchemaOptions() { + Options options = createCommonOptions(new Options()); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailedFile = + Option.builder(FAILED_FILE_ARGS) + .longOpt(FAILED_FILE_NAME) + .hasArg() + .argName(FAILED_FILE_ARGS_NAME) + .desc(FAILED_FILE_DESC) + .build(); + options.addOption(opFailedFile); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .hasArg() + .argName(BATCH_POINT_SIZE_ARGS_NAME) + .desc(BATCH_POINT_SIZE_LIMIT_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .longOpt(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArg() + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opDatabase = + Option.builder(DB_ARGS).longOpt(DB_NAME).argName(DB_ARGS).hasArg().desc(DB_DESC).build(); + options.addOption(opDatabase); + + Option opHelp = + Option.builder(Constants.HELP_ARGS).longOpt(Constants.HELP_ARGS).desc(HELP_DESC).build(); + options.addOption(opHelp); + + return options; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java new file mode 100644 index 0000000000000..54d35c4f37051 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java @@ -0,0 +1,925 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.db.utils.constant.SqlConstant; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.ImportTsFileOperation; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.thrift.annotation.Nullable; +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.tsfile.enums.TSDataType.STRING; +import static org.apache.tsfile.enums.TSDataType.TEXT; + +public abstract class AbstractDataTool { + + protected static String host; + protected static String port; + protected static String table; + protected static String endTime; + protected static String username; + protected static String password; + protected static Boolean aligned; + protected static String database; + protected static String startTime; + protected static int threadNum = 8; + protected static String targetPath; + protected static long timeout = -1; + protected static String timeZoneID; + protected static String timeFormat; + protected static String exportType; + protected static String queryCommand; + public static String fileType = null; + public static String failDir = "fail/"; + protected static String targetDirectory; + public static String timeColumn = "Time"; + protected static int linesPerFile = 10000; + public static String deviceColumn = "Device"; + protected static boolean isRemoteLoad = true; + protected static Boolean needDataTypePrinted; + protected static int batchPointSize = 100_000; + protected static boolean sqlDialectTree = true; + protected static int linesPerFailedFile = 10000; + protected static String successDir = "success/"; + protected static String timestampPrecision = "ms"; + protected static String failedFileDirectory = null; + protected static ImportTsFileOperation failOperation; + protected static ZoneId zoneId = ZoneId.systemDefault(); + protected static ImportTsFileOperation successOperation; + protected static String targetFile = Constants.DUMP_FILE_NAME_DEFAULT; + protected static final LongAdder loadFileFailedNum = new LongAdder(); + protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); + protected static final LongAdder processingLoadFailedFileSuccessfulNum = new LongAdder(); + protected static final LongAdder processingLoadSuccessfulFileSuccessfulNum = new LongAdder(); + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDataTool.class); + + protected AbstractDataTool() {} + + protected static String checkRequiredArg( + String arg, String name, CommandLine commandLine, String defaultValue) + throws ArgsErrorException { + String str = commandLine.getOptionValue(arg); + if (str == null) { + if (StringUtils.isNotBlank(defaultValue)) { + return defaultValue; + } + String msg = String.format("Required values for option '%s' not provided", name); + LOGGER.info(msg); + LOGGER.info("Use -help for more information"); + throw new ArgsErrorException(msg); + } + return str; + } + + protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException { + host = + checkRequiredArg( + Constants.HOST_ARGS, Constants.HOST_NAME, commandLine, Constants.HOST_DEFAULT_VALUE); + port = + checkRequiredArg( + Constants.PORT_ARGS, Constants.PORT_NAME, commandLine, Constants.PORT_DEFAULT_VALUE); + username = + checkRequiredArg( + Constants.USERNAME_ARGS, + Constants.USERNAME_NAME, + commandLine, + Constants.USERNAME_DEFAULT_VALUE); + password = commandLine.getOptionValue(Constants.PW_ARGS, Constants.PW_DEFAULT_VALUE); + } + + protected static void printHelpOptions( + String cmdLineHead, + String cmdLineSyntax, + HelpFormatter hf, + Options tsFileOptions, + Options csvOptions, + Options sqlOptions, + boolean printFileType) { + ioTPrinter.println( + Constants.TSFILEDB_CLI_DIVIDE + + "\n" + + cmdLineSyntax + + "\n" + + Constants.TSFILEDB_CLI_DIVIDE); + if (StringUtils.isNotBlank(cmdLineHead)) { + ioTPrinter.println(cmdLineHead); + } + final String usageName = cmdLineSyntax.replaceAll(" ", ""); + if (ObjectUtils.isNotEmpty(tsFileOptions)) { + if (printFileType) { + ioTPrinter.println( + '\n' + + Constants.FILE_TYPE_NAME + + Constants.COLON + + Constants.TSFILE_SUFFIXS + + '\n' + + Constants.TSFILEDB_CLI_DIVIDE); + } + hf.printHelp(usageName, tsFileOptions, true); + } + if (ObjectUtils.isNotEmpty(csvOptions)) { + if (printFileType) { + ioTPrinter.println( + '\n' + + Constants.FILE_TYPE_NAME + + Constants.COLON + + Constants.CSV_SUFFIXS + + '\n' + + Constants.TSFILEDB_CLI_DIVIDE); + } + hf.printHelp(usageName, csvOptions, true); + } + if (ObjectUtils.isNotEmpty(sqlOptions)) { + if (printFileType) { + ioTPrinter.println( + '\n' + + Constants.FILE_TYPE_NAME + + Constants.COLON + + Constants.SQL_SUFFIXS + + '\n' + + Constants.TSFILEDB_CLI_DIVIDE); + } + hf.printHelp(usageName, sqlOptions, true); + } + } + + protected static boolean checkTimeFormat() { + for (String format : Constants.TIME_FORMAT) { + if (timeFormat.equals(format)) { + return true; + } + } + for (String format : Constants.STRING_TIME_FORMAT) { + if (timeFormat.equals(format)) { + return true; + } + } + LOGGER.info( + "Input time format {} is not supported, " + + "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n", + timeFormat); + return false; + } + + private static void writeAndEmptyDataSet( + Session session, + List deviceIds, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); + } else { + session.insertAlignedRecords(deviceIds, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + try { + session.open(); + } catch (IoTDBConnectionException ex) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } + writeAndEmptyDataSet( + session, deviceIds, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + System.exit(1); + } finally { + deviceIds.clear(); + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + protected static void writeFailedLinesFile( + List headerNames, String failedFilePath, List> failedRecords) { + int fileIndex = 0; + int from = 0; + int failedRecordsSize = failedRecords.size(); + int restFailedRecords = failedRecordsSize; + while (from < failedRecordsSize) { + int step = Math.min(restFailedRecords, linesPerFailedFile); + writeCsvFile( + headerNames, + failedRecords.subList(from, from + step), + failedFilePath + "_" + fileIndex++); + from += step; + restFailedRecords -= step; + } + } + + /** + * if data type of timeseries is not defined in headers of schema, this method will be called to + * do type inference + * + * @param strValue + * @return + */ + protected static TSDataType typeInfer(String strValue) { + if (strValue.contains("\"")) { + return strValue.length() <= 512 + 2 ? STRING : TEXT; + } + if (isBoolean(strValue)) { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_BOOLEAN); + } else if (isTimeStamp(strValue)) { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_TIMESTAMP); + } else if (isNumber(strValue)) { + if (!strValue.contains(TsFileConstant.PATH_SEPARATOR)) { + if (isConvertFloatPrecisionLack(StringUtils.trim(strValue))) { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_LONG); + } + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_INT); + } else { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_FLOAT); + } + } else if (Constants.DATATYPE_NULL.equals(strValue) + || Constants.DATATYPE_NULL.toUpperCase().equals(strValue)) { + return null; + // "NaN" is returned if the NaN Literal is given in Parser + } else if (Constants.DATATYPE_NAN.equals(strValue)) { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_NAN); + } else if (isDate(strValue)) { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_DATE); + } else if (isBlob(strValue)) { + return Constants.TYPE_INFER_KEY_DICT.get(Constants.DATATYPE_BLOB); + } else if (strValue.length() <= 512) { + return STRING; + } else { + return TEXT; + } + } + + private static boolean isDate(String s) { + return s.equalsIgnoreCase(Constants.DATATYPE_DATE); + } + + private static boolean isTimeStamp(String s) { + return s.equalsIgnoreCase(Constants.DATATYPE_TIMESTAMP); + } + + static boolean isNumber(String s) { + if (s == null || s.equals(Constants.DATATYPE_NAN)) { + return false; + } + try { + Double.parseDouble(s); + } catch (NumberFormatException e) { + return false; + } + return true; + } + + private static boolean isBlob(String s) { + return s.length() >= 3 && s.startsWith("X'") && s.endsWith("'"); + } + + private static boolean isBoolean(String s) { + return s.equalsIgnoreCase(SqlConstant.BOOLEAN_TRUE) + || s.equalsIgnoreCase(SqlConstant.BOOLEAN_FALSE); + } + + private static boolean isConvertFloatPrecisionLack(String s) { + return Long.parseLong(s) > (2 << 24); + } + + /** + * @param value + * @param type + * @return + */ + protected static Object typeTrans(String value, TSDataType type) { + try { + switch (type) { + case TEXT: + case STRING: + if (value.startsWith("\"") && value.endsWith("\"")) { + return value.substring(1, value.length() - 1); + } + return value; + case BOOLEAN: + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + return null; + } + return Boolean.parseBoolean(value); + case INT32: + return Integer.parseInt(value); + case INT64: + return Long.parseLong(value); + case FLOAT: + return Float.parseFloat(value); + case DOUBLE: + return Double.parseDouble(value); + case TIMESTAMP: + return Long.parseLong(value); + case DATE: + return LocalDate.parse(value); + case BLOB: + return new Binary(parseHexStringToByteArray(value.replaceFirst("0x", ""))); + default: + return null; + } + } catch (NumberFormatException e) { + return null; + } + } + + private static byte[] parseHexStringToByteArray(String hexString) { + byte[] bytes = new byte[hexString.length() / 2]; + for (int i = 0; i < hexString.length(); i += 2) { + int value = Integer.parseInt(hexString.substring(i, i + 2), 16); + bytes[i / 2] = (byte) value; + } + return bytes; + } + + protected static long parseTimestamp(String str) { + long timestamp; + try { + timestamp = Long.parseLong(str); + } catch (NumberFormatException e) { + timestamp = DateTimeUtils.convertDatetimeStrToLong(str, zoneId, timestampPrecision); + } + return timestamp; + } + + /** + * return the TSDataType + * + * @param typeStr + * @return + */ + protected static TSDataType getType(String typeStr) { + try { + return TSDataType.valueOf(typeStr); + } catch (Exception e) { + return null; + } + } + + /** + * return the ColumnCategory + * + * @param typeStr + * @return + */ + protected static ColumnCategory getColumnCategory(String typeStr) { + if (StringUtils.isNotBlank(typeStr)) { + try { + return ColumnCategory.valueOf(typeStr); + } catch (Exception e) { + return null; + } + } + return null; + } + + /** + * if the data is aligned by device, the data will be written by this method. + * + * @param headerNames the header names of CSV file + * @param records the records of CSV file + * @param failedFilePath the directory to save the failed files + */ + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + protected static void writeDataAlignedByDevice( + Session session, List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + HashMap headerTypeMap = new HashMap<>(); + HashMap headerNameMap = new HashMap<>(); + parseHeaders(headerNames, null, headerTypeMap, headerNameMap); + + AtomicReference deviceName = new AtomicReference<>(null); + + HashSet typeQueriedDevice = new HashSet<>(); + + // the data that interface need + List times = new ArrayList<>(); + List> typesList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + + AtomicInteger pointSize = new AtomicInteger(0); + + List> failedRecords = new ArrayList<>(); + + records.forEach( + recordObj -> { + // only run in first record + if (deviceName.get() == null) { + deviceName.set(recordObj.get(1)); + } else if (!Objects.equals(deviceName.get(), recordObj.get(1))) { + // if device changed + writeAndEmptyDataSet( + session, deviceName.get(), times, typesList, valuesList, measurementsList, 3); + deviceName.set(recordObj.get(1)); + pointSize.set(0); + } else if (pointSize.get() >= batchPointSize) { + // insert a batch + writeAndEmptyDataSet( + session, deviceName.get(), times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + // the data of the record + ArrayList types = new ArrayList<>(); + ArrayList values = new ArrayList<>(); + ArrayList measurements = new ArrayList<>(); + + AtomicReference isFail = new AtomicReference<>(false); + + // read data from record + for (Map.Entry headerNameEntry : headerNameMap.entrySet()) { + // headerNameWithoutType is equal to headerName if the CSV column do not have data type. + String headerNameWithoutType = headerNameEntry.getKey(); + String headerName = headerNameEntry.getValue(); + String value = recordObj.get(headerName); + if (!"".equals(value)) { + TSDataType type; + // Get the data type directly if the CSV column have data type. + if (!headerTypeMap.containsKey(headerNameWithoutType)) { + boolean hasResult = false; + // query the data type in iotdb + if (!typeQueriedDevice.contains(deviceName.get())) { + if (headerTypeMap.isEmpty()) { + Set devices = new HashSet<>(); + devices.add(deviceName.get()); + queryType(session, devices, headerTypeMap, deviceColumn); + } + typeQueriedDevice.add(deviceName.get()); + } + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(headerNameWithoutType, type); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), headerNameWithoutType, value); + isFail.set(true); + } + } + type = headerTypeMap.get(headerNameWithoutType); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail.set(true); + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), headerNameWithoutType, value, type); + } else { + values.add(valueTrans); + measurements.add(headerNameWithoutType); + types.add(type); + pointSize.getAndIncrement(); + } + } + } + } + if (Boolean.TRUE.equals(isFail.get())) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + if (!measurements.isEmpty()) { + times.add(parseTimestamp(recordObj.get(timeColumn))); + typesList.add(types); + valuesList.add(values); + measurementsList.add(measurements); + } + }); + if (!times.isEmpty()) { + writeAndEmptyDataSet( + session, deviceName.get(), times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + ioTPrinter.println("Import completely!"); + } + + /** + * if the data is aligned by time, the data will be written by this method. + * + * @param headerNames the header names of CSV file + * @param records the records of CSV file + * @param failedFilePath the directory to save the failed files + */ + @SuppressWarnings("squid:S3776") + protected static void writeDataAlignedByTime( + Session session, List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + HashMap> deviceAndMeasurementNames = new HashMap<>(); + HashMap headerTypeMap = new HashMap<>(); + HashMap headerNameMap = new HashMap<>(); + parseHeaders(headerNames, deviceAndMeasurementNames, headerTypeMap, headerNameMap); + + Set devices = deviceAndMeasurementNames.keySet(); + if (headerTypeMap.isEmpty()) { + queryType(session, devices, headerTypeMap, "Time"); + } + + List deviceIds = new ArrayList<>(); + List times = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + List> typesList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + + AtomicReference hasStarted = new AtomicReference<>(false); + AtomicInteger pointSize = new AtomicInteger(0); + + ArrayList> failedRecords = new ArrayList<>(); + + records.forEach( + recordObj -> { + if (Boolean.FALSE.equals(hasStarted.get())) { + hasStarted.set(true); + } else if (pointSize.get() >= batchPointSize) { + writeAndEmptyDataSet( + session, deviceIds, times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + boolean isFail = false; + + for (Map.Entry> entry : deviceAndMeasurementNames.entrySet()) { + String deviceId = entry.getKey(); + List measurementNames = entry.getValue(); + ArrayList types = new ArrayList<>(); + ArrayList values = new ArrayList<>(); + ArrayList measurements = new ArrayList<>(); + for (String measurement : measurementNames) { + String header = deviceId + "." + measurement; + String value = recordObj.get(headerNameMap.get(header)); + if (!"".equals(value)) { + TSDataType type; + if (!headerTypeMap.containsKey(header)) { + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(header, type); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), header, value); + isFail = true; + } + } + type = headerTypeMap.get(header); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail = true; + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), header, value, type); + } else { + measurements.add(header.replace(deviceId + '.', "")); + types.add(type); + values.add(valueTrans); + pointSize.getAndIncrement(); + } + } + } + } + if (!measurements.isEmpty()) { + times.add(parseTimestamp(recordObj.get(timeColumn))); + deviceIds.add(deviceId); + typesList.add(types); + valuesList.add(values); + measurementsList.add(measurements); + } + } + if (isFail) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + }); + if (!deviceIds.isEmpty()) { + writeAndEmptyDataSet(session, deviceIds, times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + if (Boolean.TRUE.equals(hasStarted.get())) { + ioTPrinter.println("Import completely!"); + } else { + ioTPrinter.println("No records!"); + } + } + + private static void writeAndEmptyDataSet( + Session session, + String device, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + session.insertRecordsOfOneDevice(device, times, measurementsList, typesList, valuesList); + } else { + session.insertAlignedRecordsOfOneDevice( + device, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + try { + session.open(); + } catch (IoTDBConnectionException ex) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } + writeAndEmptyDataSet( + session, device, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + System.exit(1); + } finally { + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + /** + * query data type of timeseries from IoTDB + * + * @param deviceNames + * @param headerTypeMap + * @param alignedType + * @throws IoTDBConnectionException + * @throws StatementExecutionException + */ + private static void queryType( + Session session, + Set deviceNames, + HashMap headerTypeMap, + String alignedType) { + for (String deviceName : deviceNames) { + String sql = "show timeseries " + deviceName + ".*"; + SessionDataSet sessionDataSet; + try { + sessionDataSet = session.executeQueryStatement(sql); + int tsIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); + int dtIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + String timeseries = fields.get(tsIndex).getStringValue(); + String dataType = fields.get(dtIndex).getStringValue(); + if (Objects.equals(alignedType, "Time")) { + headerTypeMap.put(timeseries, getType(dataType)); + } else if (Objects.equals(alignedType, deviceColumn)) { + String[] split = PathUtils.splitPathToDetachedNodes(timeseries); + String measurement = split[split.length - 1]; + headerTypeMap.put(measurement, getType(dataType)); + } + } + } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { + ioTPrinter.println( + "Meet error when query the type of timeseries because " + e.getMessage()); + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + System.exit(1); + } + } + } + + @SuppressWarnings( + "squid:S135") // ignore for loops should not contain more than a single "break" or "continue" + // statement + protected static void parseHeaders( + List headerNames, + @Nullable HashMap> deviceAndMeasurementNames, + HashMap headerTypeMap, + HashMap headerNameMap) + throws IllegalPathException { + String regex = "(?<=\\()\\S+(?=\\))"; + Pattern pattern = Pattern.compile(regex); + for (String headerName : headerNames) { + if ("Time".equalsIgnoreCase(filterBomHeader(headerName))) { + timeColumn = headerName; + continue; + } else if ("Device".equalsIgnoreCase(headerName)) { + deviceColumn = headerName; + continue; + } + Matcher matcher = pattern.matcher(headerName); + String type; + String headerNameWithoutType; + if (matcher.find()) { + type = matcher.group(); + headerNameWithoutType = headerName.replace("(" + type + ")", "").replaceAll("\\s+", ""); + headerNameMap.put(headerNameWithoutType, headerName); + headerTypeMap.put(headerNameWithoutType, getType(type)); + } else { + headerNameWithoutType = headerName; + headerNameMap.put(headerName, headerName); + } + String[] split = PathUtils.splitPathToDetachedNodes(headerNameWithoutType); + String measurementName = split[split.length - 1]; + String deviceName = StringUtils.join(Arrays.copyOfRange(split, 0, split.length - 1), '.'); + if (deviceAndMeasurementNames != null) { + deviceAndMeasurementNames.putIfAbsent(deviceName, new ArrayList<>()); + deviceAndMeasurementNames.get(deviceName).add(measurementName); + } + } + } + + protected static String filterBomHeader(String s) { + byte[] bom = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; + byte[] bytes = Arrays.copyOf(s.getBytes(), 3); + if (Arrays.equals(bom, bytes)) { + return s.substring(1); + } + return s; + } + + /** + * read data from the CSV file + * + * @param path + * @return CSVParser csv parser + * @throws IOException when reading the csv file failed. + */ + protected static CSVParser readCsvFile(String path) throws IOException { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setQuote('`') + .setEscape('\\') + .setIgnoreEmptyLines(true) + .build() + .parse(new InputStreamReader(new FileInputStream(path))); + } + + /** + * write data to CSV file. + * + * @param headerNames the header names of CSV file + * @param records the records of CSV file + * @param filePath the directory to save the file + */ + public static Boolean writeCsvFile( + List headerNames, List> records, String filePath) { + try { + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(filePath); + if (headerNames != null) { + csvPrinterWrapper.printRecord(headerNames); + } + for (List CsvRecord : records) { + csvPrinterWrapper.printRecord(CsvRecord); + } + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + return true; + } catch (IOException e) { + ioTPrinter.printException(e); + return false; + } + } + + static class CSVPrinterWrapper { + private final String filePath; + private final CSVFormat csvFormat; + private CSVPrinter csvPrinter; + + public CSVPrinterWrapper(String filePath) { + this.filePath = filePath; + this.csvFormat = + CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setEscape('\\') + .setQuoteMode(QuoteMode.NONE) + .build(); + } + + public void printRecord(final Iterable values) throws IOException { + if (csvPrinter == null) { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } + csvPrinter.printRecord(values); + } + + public void print(Object value) { + if (csvPrinter == null) { + try { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } catch (IOException e) { + ioTPrinter.printException(e); + return; + } + } + try { + csvPrinter.print(value); + } catch (IOException e) { + ioTPrinter.printException(e); + } + } + + public void println() throws IOException { + csvPrinter.println(); + } + + public void close() throws IOException { + if (csvPrinter != null) { + csvPrinter.close(); + } + } + + public void flush() throws IOException { + if (csvPrinter != null) { + csvPrinter.flush(); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractExportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractExportData.java new file mode 100644 index 0000000000000..f1d506a72b49c --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractExportData.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.thrift.TException; + +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractExportData extends AbstractDataTool { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + public abstract void init() + throws IoTDBConnectionException, StatementExecutionException, TException; + + public abstract void exportBySql(String sql, int index); + + protected static void legalCheck(String sql) { + String aggregatePattern = + "\\b(count|sum|avg|extreme|max_value|min_value|first_value|last_value|max_time|min_time|stddev|stddev_pop|stddev_samp|variance|var_pop|var_samp|max_by|min_by)\\b\\s*\\("; + Pattern pattern = Pattern.compile(aggregatePattern, Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(sql); + if (matcher.find()) { + ioTPrinter.println("The sql you entered is invalid, please don't use aggregate query."); + System.exit(Constants.CODE_ERROR); + } + } + + protected static String timeTrans(Long time) { + switch (timeFormat) { + case "default": + return RpcUtils.parseLongToDateWithPrecision( + DateTimeFormatter.ISO_OFFSET_DATE_TIME, time, zoneId, timestampPrecision); + case "timestamp": + case "long": + case "number": + return String.valueOf(time); + default: + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), zoneId) + .format(DateTimeFormatter.ofPattern(timeFormat)); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractImportData.java new file mode 100644 index 0000000000000..fa80c8feb2014 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractImportData.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.tsfile.ImportTsFileScanTool; + +import java.io.File; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public abstract class AbstractImportData extends AbstractDataTool implements Runnable { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + public abstract void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException; + + @Override + public void run() { + String filePath = ""; + File file = null; + try { + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + while ((filePath = ImportTsFileScanTool.pollFromQueue()) != null) { + file = new File(filePath); + if (file.getName().endsWith(Constants.TSFILE_SUFFIXS)) { + importFromTsFile(file); + } + } + } else { + while ((filePath = ImportDataScanTool.pollFromQueue()) != null) { + file = new File(filePath); + if (file.getName().endsWith(Constants.SQL_SUFFIXS)) { + importFromSqlFile(file); + } else { + importFromCsvFile(file); + } + } + } + } catch (Exception e) { + ioTPrinter.println( + String.format("[%s] - Unexpected error occurred: %s", file.getName(), e.getMessage())); + } + } + + protected abstract Runnable getAsyncImportRunnable(); + + protected class ThreadManager { + public void asyncImportDataFiles() { + List list = new ArrayList<>(threadNum); + for (int i = 0; i < threadNum; i++) { + Thread thread = new Thread(getAsyncImportRunnable()); + thread.start(); + list.add(thread); + } + list.forEach( + thread -> { + try { + thread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + ioTPrinter.println("ImportData thread join interrupted: " + e.getMessage()); + } + }); + ioTPrinter.println(Constants.IMPORT_COMPLETELY); + } + } + + public static void init(AbstractImportData instance) { + instance.new ThreadManager().asyncImportDataFiles(); + } + + protected abstract void importFromSqlFile(File file); + + protected abstract void importFromTsFile(File file); + + protected abstract void importFromCsvFile(File file); + + protected void processSuccessFile(String file) { + loadFileSuccessfulNum.increment(); + if (fileType.equalsIgnoreCase(Constants.TSFILE_SUFFIXS)) { + try { + processingFile(file, true); + processingLoadSuccessfulFileSuccessfulNum.increment(); + ioTPrinter.println("Processed success file [ " + file + " ] successfully!"); + } catch (final Exception processSuccessException) { + ioTPrinter.println( + "Failed to process success file [ " + + file + + " ]: " + + processSuccessException.getMessage()); + } + } + } + + protected void processingFile(final String file, final boolean isSuccess) { + final String relativePath = file.substring(ImportTsFileScanTool.getSourceFullPathLength() + 1); + final Path sourcePath = Paths.get(file); + + final String target = + isSuccess + ? successDir + : failDir + File.separator + relativePath.replace(File.separator, "_"); + final Path targetPath = Paths.get(target); + + final String RESOURCE = ".resource"; + Path sourceResourcePath = Paths.get(sourcePath + RESOURCE); + sourceResourcePath = Files.exists(sourceResourcePath) ? sourceResourcePath : null; + final Path targetResourcePath = Paths.get(target + RESOURCE); + + final String MODS = ".mods"; + Path sourceModsPath = Paths.get(sourcePath + MODS); + sourceModsPath = Files.exists(sourceModsPath) ? sourceModsPath : null; + final Path targetModsPath = Paths.get(target + MODS); + + switch (isSuccess ? successOperation : failOperation) { + case DELETE: + { + try { + Files.deleteIfExists(sourcePath); + if (null != sourceResourcePath) { + Files.deleteIfExists(sourceResourcePath); + } + if (null != sourceModsPath) { + Files.deleteIfExists(sourceModsPath); + } + } catch (final Exception e) { + ioTPrinter.println(String.format("Failed to delete file: %s", e.getMessage())); + } + break; + } + case CP: + { + try { + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + if (null != sourceResourcePath) { + Files.copy( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + ioTPrinter.println(String.format("Failed to copy file: %s", e.getMessage())); + } + break; + } + case HARDLINK: + { + try { + Files.createLink(targetPath, sourcePath); + } catch (FileAlreadyExistsException e) { + ioTPrinter.println("Hardlink already exists: " + e.getMessage()); + } catch (final Exception e) { + try { + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (final Exception copyException) { + ioTPrinter.println( + String.format("Failed to copy file: %s", copyException.getMessage())); + } + } + + try { + if (null != sourceResourcePath) { + Files.copy( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + ioTPrinter.println( + String.format("Failed to copy resource or mods file: %s", e.getMessage())); + } + break; + } + case MV: + { + try { + Files.move(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + if (null != sourceResourcePath) { + Files.move( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.move(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + ioTPrinter.println(String.format("Failed to move file: %s", e.getMessage())); + } + break; + } + default: + break; + } + } + + protected void processFailFile(final String filePath, final Exception e) { + try { + if (Objects.nonNull(e.getMessage()) && e.getMessage().contains("memory")) { + ioTPrinter.println( + "Rejecting file [ " + filePath + " ] due to memory constraints, will retry later."); + ImportTsFileScanTool.putToQueue(filePath); + return; + } + + loadFileFailedNum.increment(); + ioTPrinter.println("Failed to import [ " + filePath + " ] file: " + e.getMessage()); + + try { + processingFile(filePath, false); + processingLoadFailedFileSuccessfulNum.increment(); + ioTPrinter.println("Processed fail file [ " + filePath + " ] successfully!"); + } catch (final Exception processFailException) { + ioTPrinter.println( + "Failed to process fail file [ " + + filePath + + " ]: " + + processFailException.getMessage()); + } + } catch (final InterruptedException e1) { + ioTPrinter.println("Unexpected error occurred: " + e1.getMessage()); + Thread.currentThread().interrupt(); + } catch (final Exception e1) { + ioTPrinter.println("Unexpected error occurred: " + e1.getMessage()); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java new file mode 100644 index 0000000000000..b31bc8d90b989 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.type.ExitType; +import org.apache.iotdb.cli.utils.CliContext; +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.cli.utils.JlineUtils; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.OptionsUtil; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.thrift.TException; +import org.jline.reader.LineReader; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +/** + * Export CSV file. + * + * @version 1.0.0 20170719 + */ +public class ExportData extends AbstractDataTool { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + @SuppressWarnings({ + "squid:S3776", + "squid:S2093" + }) // Suppress high Cognitive Complexity warning, ignore try-with-resources + /* main function of export csv tool. */ + public static void main(String[] args) { + OptionsUtil.setIsImport(false); + Options helpOptions = OptionsUtil.createHelpOptions(); + Options tsFileOptions = OptionsUtil.createExportTsFileOptions(); + Options csvOptions = OptionsUtil.createExportCsvOptions(); + Options sqlOptions = OptionsUtil.createExportSqlOptions(); + HelpFormatter hf = new HelpFormatter(); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); + hf.setOptionComparator(null); // avoid reordering + hf.setWidth(Constants.MAX_HELP_CONSOLE_WIDTH); + + if (args == null || args.length == 0) { + printHelpOptions( + Constants.EXPORT_CLI_HEAD, + Constants.EXPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + try { + commandLine = parser.parse(helpOptions, args, true); + } catch (ParseException e) { + printHelpOptions( + Constants.EXPORT_CLI_HEAD, + Constants.EXPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + final List argList = Arrays.asList(args); + int helpIndex = argList.indexOf(Constants.MINUS + Constants.HELP_ARGS); + int sql_dialect = argList.indexOf(Constants.MINUS + Constants.SQL_DIALECT_ARGS); // -sql_dialect + if (sql_dialect >= 0 + && !Constants.SQL_DIALECT_VALUE_TREE.equalsIgnoreCase(argList.get(sql_dialect + 1))) { + final String sqlDialectValue = argList.get(sql_dialect + 1); + if (Constants.SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) { + sqlDialectTree = false; + csvOptions = OptionsUtil.createTableExportCsvOptions(); + tsFileOptions = OptionsUtil.createTableExportTsFileOptions(); + sqlOptions = OptionsUtil.createTableExportSqlOptions(); + } else { + ioTPrinter.println(String.format("sql_dialect %s is not support", sqlDialectValue)); + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + } + int ftIndex = argList.indexOf(Constants.MINUS + Constants.FILE_TYPE_ARGS); + if (ftIndex < 0) { + ftIndex = argList.indexOf(Constants.MINUS + Constants.FILE_TYPE_NAME); + } + if (helpIndex >= 0) { + fileType = argList.get(helpIndex + 1); + if (StringUtils.isNotBlank(fileType)) { + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, Constants.EXPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); + } else if (Constants.CSV_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, Constants.EXPORT_CLI_PREFIX, hf, null, csvOptions, null, false); + } else if (Constants.SQL_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, Constants.EXPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); + } else { + ioTPrinter.println(String.format("File type %s is not support", fileType)); + printHelpOptions( + Constants.EXPORT_CLI_HEAD, + Constants.EXPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + } + } else { + printHelpOptions( + Constants.EXPORT_CLI_HEAD, + Constants.EXPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + } + System.exit(Constants.CODE_ERROR); + } else if (ftIndex >= 0) { + fileType = argList.get(ftIndex + 1); + if (StringUtils.isNotBlank(fileType)) { + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(tsFileOptions, args, true); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions( + null, Constants.EXPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); + System.exit(Constants.CODE_ERROR); + } + } else if (Constants.CSV_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(csvOptions, args, true); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, Constants.EXPORT_CLI_PREFIX, hf, null, csvOptions, null, false); + System.exit(Constants.CODE_ERROR); + } + } else if (Constants.SQL_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(sqlOptions, args, true); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, Constants.EXPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); + System.exit(Constants.CODE_ERROR); + } + } else { + ioTPrinter.println(String.format("File type %s is not support", fileType)); + printHelpOptions( + Constants.EXPORT_CLI_HEAD, + Constants.EXPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + } else { + printHelpOptions( + Constants.EXPORT_CLI_HEAD, + Constants.EXPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + } else { + ioTPrinter.println( + String.format( + "Invalid args: Required values for option '%s' not provided", + Constants.FILE_TYPE_NAME)); + System.exit(Constants.CODE_ERROR); + } + int exitCode = Constants.CODE_OK; + try { + parseBasicParams(commandLine); + parseSpecialParams(commandLine); + if (!checkTimeFormat()) { + System.exit(Constants.CODE_ERROR); + } + AbstractExportData exportData; + exportData = new ExportDataTree(); + exportData.init(); + if (!sqlDialectTree) { + exportData = new ExportDataTable(); + exportData.init(); + } + if (sqlDialectTree && queryCommand == null) { + LineReader lineReader = + JlineUtils.getLineReader( + new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), + username, + host, + port); + String sql = lineReader.readLine(Constants.EXPORT_CLI_PREFIX + "> please input query: "); + ioTPrinter.println(sql); + String[] values = sql.trim().split(";"); + for (int i = 0; i < values.length; i++) { + exportData.exportBySql(values[i], i); + } + } else { + exportData.exportBySql(queryCommand, 0); + } + } catch (IOException e) { + ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); + exitCode = Constants.CODE_ERROR; + } catch (ArgsErrorException e) { + ioTPrinter.println("Invalid args: " + e.getMessage()); + exitCode = Constants.CODE_ERROR; + } catch (IoTDBConnectionException | StatementExecutionException e) { + ioTPrinter.println("Connect failed because " + e.getMessage()); + exitCode = Constants.CODE_ERROR; + } catch (TException e) { + ioTPrinter.println( + "Can not get the timestamp precision from server because " + e.getMessage()); + exitCode = Constants.CODE_ERROR; + } + System.exit(exitCode); + } + + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { + targetDirectory = + checkRequiredArg(Constants.TARGET_DIR_ARGS, Constants.TARGET_DIR_NAME, commandLine, null); + targetFile = commandLine.getOptionValue(Constants.TARGET_FILE_ARGS); + needDataTypePrinted = Boolean.valueOf(commandLine.getOptionValue(Constants.DATA_TYPE_ARGS)); + queryCommand = commandLine.getOptionValue(Constants.QUERY_COMMAND_ARGS); + exportType = commandLine.getOptionValue(Constants.FILE_TYPE_ARGS); + String timeoutString = commandLine.getOptionValue(Constants.TIMEOUT_ARGS); + if (timeoutString != null) { + timeout = Long.parseLong(timeoutString); + } + if (needDataTypePrinted == null) { + needDataTypePrinted = true; + } + if (targetFile == null) { + targetFile = Constants.DUMP_FILE_NAME_DEFAULT; + } + timeFormat = commandLine.getOptionValue(Constants.TIME_FORMAT_ARGS); + if (timeFormat == null) { + timeFormat = "default"; + } + timeZoneID = commandLine.getOptionValue(Constants.TIME_ZONE_ARGS); + if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { + targetDirectory += File.separator; + } + final File file = new File(targetDirectory); + if (!file.isDirectory() && !file.mkdirs()) { + ioTPrinter.println(String.format("Failed to create directories %s", targetDirectory)); + System.exit(Constants.CODE_ERROR); + } + if (commandLine.getOptionValue(Constants.LINES_PER_FILE_ARGS) != null) { + linesPerFile = Integer.parseInt(commandLine.getOptionValue(Constants.LINES_PER_FILE_ARGS)); + } + if (commandLine.getOptionValue(Constants.ALIGNED_ARGS) != null) { + aligned = Boolean.valueOf(commandLine.getOptionValue(Constants.ALIGNED_ARGS)); + } + if (commandLine.getOptionValue(Constants.DB_ARGS) != null) { + database = commandLine.getOptionValue(Constants.DB_ARGS).toLowerCase(); + if (ObjectUtils.isNotEmpty(database) && "information_schema".equalsIgnoreCase(database)) { + ioTPrinter.println( + String.format("Does not support exporting system databases %s", database)); + System.exit(Constants.CODE_ERROR); + } + } + if (commandLine.getOptionValue(Constants.TABLE_ARGS) != null) { + table = commandLine.getOptionValue(Constants.TABLE_ARGS).toLowerCase(); + } + if (commandLine.getOptionValue(Constants.START_TIME_ARGS) != null) { + startTime = commandLine.getOptionValue(Constants.START_TIME_ARGS); + } + if (commandLine.getOptionValue(Constants.END_TIME_ARGS) != null) { + endTime = commandLine.getOptionValue(Constants.END_TIME_ARGS); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java new file mode 100644 index 0000000000000..afe7f1dbd48ee --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.TableSessionBuilder; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ExportDataTable extends AbstractExportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static ITableSession tableSession; + private static List tables = new ArrayList<>(); + + @Override + public void init() throws IoTDBConnectionException, StatementExecutionException { + tableSession = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(host + ":" + port)) + .username(username) + .password(password) + .database(database) + .build(); + SessionDataSet sessionDataSet = tableSession.executeQueryStatement("show databases", timeout); + List databases = new ArrayList<>(); + while (sessionDataSet.hasNext()) { + databases.add(sessionDataSet.next().getField(0).getStringValue()); + } + if (CollectionUtils.isEmpty(databases) || !databases.contains(database)) { + ioTPrinter.println(String.format(Constants.TARGET_DATABASE_NOT_EXIST_MSG, database)); + System.exit(1); + } + sessionDataSet = tableSession.executeQueryStatement("show tables", timeout); + while (sessionDataSet.hasNext()) { + tables.add(sessionDataSet.next().getField(0).getStringValue()); + } + if (CollectionUtils.isEmpty(tables) + || (ObjectUtils.isNotEmpty(table) && !tables.contains(table))) { + ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, table)); + System.exit(1); + } + if (ObjectUtils.isNotEmpty(table)) { + tables.clear(); + tables.add(table); + } + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + sessionDataSet.close(); + } + } + + @Override + public void exportBySql(String sql, int index) { + List exportSql = new ArrayList<>(); + if (StringUtils.isNotBlank(sql)) { + if (Constants.SQL_SUFFIXS.equalsIgnoreCase(exportType) + || Constants.TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + legalCheck(sql); + } + exportSql.add(sql); + } else { + StringBuilder sqlBuilder; + for (String table : tables) { + sqlBuilder = new StringBuilder("select * from ").append(table); + if (StringUtils.isNotBlank(startTime) || StringUtils.isNotBlank(endTime)) { + sqlBuilder.append(" where "); + if (StringUtils.isNotBlank(startTime)) { + sqlBuilder.append("time >= ").append(startTime); + } + if (StringUtils.isNotBlank(startTime) && StringUtils.isNotBlank(endTime)) { + sqlBuilder.append(" and "); + } + if (StringUtils.isNotBlank(endTime)) { + sqlBuilder.append("time <= ").append(endTime); + } + } + exportSql.add(sqlBuilder.toString()); + } + } + for (int i = 0; i < exportSql.size(); i++) { + String path = targetDirectory + targetFile + i; + String table = tables.get(i); + try (SessionDataSet sessionDataSet = + tableSession.executeQueryStatement(exportSql.get(i), timeout)) { + if (Constants.SQL_SUFFIXS.equalsIgnoreCase(exportType)) { + exportToSqlFile(sessionDataSet, table, path); + } else if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + long start = System.currentTimeMillis(); + boolean isComplete = exportToTsFile(sessionDataSet, path + ".tsfile", table); + if (isComplete) { + long end = System.currentTimeMillis(); + ioTPrinter.println("Export completely!cost: " + (end - start) + " ms."); + } + } else { + exportToCsvFile(sessionDataSet, path); + } + sessionDataSet.closeOperationHandle(); + ioTPrinter.println(Constants.EXPORT_COMPLETELY); + } catch (StatementExecutionException + | IoTDBConnectionException + | IOException + | WriteProcessException e) { + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); + } + } + } + + private void exportToSqlFile(SessionDataSet sessionDataSet, String table, String filePath) + throws IOException, IoTDBConnectionException, StatementExecutionException { + StringBuilder sqlBuilder; + List headers = sessionDataSet.getColumnNames(); + String prevSql = "insert into " + table + "(" + StringUtils.join(headers, ",") + ") values("; + final List columnTypes = sessionDataSet.getColumnTypes(); + boolean hasNext = sessionDataSet.hasNext(); + int fileIndex = 0; + while (hasNext) { + final String finalFilePath = filePath + "_" + fileIndex + ".sql"; + int countLine = 0; + try (FileWriter writer = new FileWriter(finalFilePath)) { + while (countLine++ < linesPerFile && hasNext) { + RowRecord rowRecord = sessionDataSet.next(); + sqlBuilder = new StringBuilder(); + List fields = rowRecord.getFields(); + sqlBuilder.append(prevSql); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sqlBuilder.append(","); + } + final TSDataType type = getType(columnTypes.get(i)); + if (TSDataType.TEXT.equals(type) || TSDataType.STRING.equals(type)) { + sqlBuilder.append("\'").append(fields.get(i).getObjectValue(type)).append("\'"); + } else { + sqlBuilder.append(fields.get(i).getObjectValue(type)); + } + } + sqlBuilder.append(");\n"); + writer.write(sqlBuilder.toString()); + hasNext = sessionDataSet.hasNext(); + } + writer.flush(); + fileIndex++; + } + } + } + + private Boolean exportToTsFile(SessionDataSet sessionDataSet, String filePath, String table) + throws IOException, + IoTDBConnectionException, + StatementExecutionException, + WriteProcessException { + List columnNamesRaw = sessionDataSet.getColumnNames(); + List columnTypesRaw = + sessionDataSet.getColumnTypes().stream().map(t -> getType(t)).collect(Collectors.toList()); + File f = FSFactoryProducer.getFSFactory().getFile(filePath); + if (f.exists()) { + Files.delete(f.toPath()); + } + boolean isEmpty = false; + Map deviceColumnIndices = new HashMap<>(); + List columnSchemas = collectSchemas(columnNamesRaw, deviceColumnIndices, table); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder() + .file(f) + .tableSchema(new org.apache.tsfile.file.metadata.TableSchema(table, columnSchemas)) + .memoryThreshold(Constants.memoryThreshold) + .build()) { + List columnNames = new ArrayList<>(columnNamesRaw); + List columnTypes = new ArrayList<>(columnTypesRaw); + int timeIndex = columnNamesRaw.indexOf(timeColumn.toLowerCase()); + if (timeIndex >= 0) { + columnNames.remove(timeIndex); + columnTypes.remove(timeIndex); + } + Tablet tablet = new Tablet(columnNames, columnTypes); + if (ObjectUtils.isNotEmpty(tablet)) { + writeWithTablets(sessionDataSet, tablet, deviceColumnIndices, tsFileWriter); + } else { + isEmpty = true; + } + } + if (isEmpty) { + ioTPrinter.println("!!!Warning:Tablet is empty,no data can be exported."); + return false; + } + return true; + } + + private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, IoTDBConnectionException, StatementExecutionException { + List headers = sessionDataSet.getColumnNames(); + int fileIndex = 0; + boolean hasNext = sessionDataSet.hasNext(); + while (hasNext) { + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); + csvPrinterWrapper.printRecord(headers); + int i = 0; + while (i++ < linesPerFile) { + RowRecord rowRecord = sessionDataSet.next(); + if (rowRecord.getTimestamp() != 0) { + csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); + } + rowRecord + .getFields() + .forEach( + field -> { + String fieldStringValue = field.getStringValue(); + if (!"null".equals(field.getStringValue())) { + if ((field.getDataType() == TSDataType.TEXT + || field.getDataType() == TSDataType.STRING)) { + fieldStringValue = "\"" + fieldStringValue + "\""; + } + csvPrinterWrapper.print(fieldStringValue); + } else { + csvPrinterWrapper.print(""); + } + }); + csvPrinterWrapper.println(); + // 检查下一行是否存在 + hasNext = sessionDataSet.hasNext(); + if (!hasNext) { + break; + } + } + fileIndex++; + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + } + } + + private static void writeWithTablets( + SessionDataSet sessionDataSet, + Tablet tablet, + Map deviceColumnIndices, + ITsFileWriter tsFileWriter) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp( + rowIndex, fields.get(deviceColumnIndices.get(timeColumn.toLowerCase())).getLongV()); + List schemas = tablet.getSchemas(); + for (int i = 0; i < schemas.size(); i++) { + IMeasurementSchema measurementSchema = schemas.get(i); + // -1 for time not in fields + final String measurementName = measurementSchema.getMeasurementName(); + if (timeColumn.equalsIgnoreCase(measurementName)) { + continue; + } + Object value = + fields + .get(deviceColumnIndices.get(measurementName)) + .getObjectValue(measurementSchema.getType()); + tablet.addValue(measurementName, rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writeToTsFile(tsFileWriter, tablet); + tablet.initBitMaps(); + tablet.reset(); + } + } + if (tablet.getRowSize() != 0) { + writeToTsFile(tsFileWriter, tablet); + } + } + + private static void writeToTsFile(ITsFileWriter tsFileWriter, Tablet tablet) + throws IOException, WriteProcessException { + tsFileWriter.write(tablet); + } + + private List collectSchemas( + List columnNames, Map deviceColumnIndices, String table) + throws IoTDBConnectionException, StatementExecutionException { + List columnSchemas = new ArrayList<>(); + SessionDataSet sessionDataSet = tableSession.executeQueryStatement("describe " + table); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + final String columnName = rowRecord.getField(0).getStringValue(); + if (timeColumn.equalsIgnoreCase(columnName)) { + continue; + } + columnSchemas.add( + new ColumnSchemaBuilder() + .name(columnName) + .dataType(getType(rowRecord.getField(1).getStringValue())) + .category(getColumnCategory(rowRecord.getField(2).getStringValue())) + .build()); + } + for (int i = 0; i < columnNames.size(); i++) { + deviceColumnIndices.put(columnNames.get(i), i); + } + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + sessionDataSet.close(); + } + return columnSchemas; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java new file mode 100644 index 0000000000000..c9ff2d28dd1ba --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; + +public class ExportDataTree extends AbstractExportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static Session session; + + @Override + public void init() throws IoTDBConnectionException, StatementExecutionException, TException { + session = new Session(host, Integer.parseInt(port), username, password); + session.open(false); + timestampPrecision = session.getTimestampPrecision(); + if (timeZoneID != null) { + session.setTimeZone(timeZoneID); + } + zoneId = ZoneId.of(session.getTimeZone()); + } + + @Override + public void exportBySql(String sql, int index) { + if (Constants.SQL_SUFFIXS.equalsIgnoreCase(exportType) + || Constants.TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + legalCheck(sql); + } + final String path = targetDirectory + targetFile + index; + try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql, timeout)) { + if (Constants.SQL_SUFFIXS.equalsIgnoreCase(exportType)) { + exportToSqlFile(sessionDataSet, path); + } else if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + long start = System.currentTimeMillis(); + boolean isComplete = exportToTsFile(sessionDataSet, path + ".tsfile"); + if (isComplete) { + long end = System.currentTimeMillis(); + ioTPrinter.println("Export completely!cost: " + (end - start) + " ms."); + } + } else { + List headers = new ArrayList<>(); + List names = sessionDataSet.getColumnNames(); + List types = sessionDataSet.getColumnTypes(); + if (Boolean.TRUE.equals(needDataTypePrinted)) { + for (int i = 0; i < names.size(); i++) { + if (!"Time".equals(names.get(i)) && !"Device".equals(names.get(i))) { + headers.add(String.format("%s(%s)", names.get(i), types.get(i))); + } else { + headers.add(names.get(i)); + } + } + } else { + headers.addAll(names); + } + exportToCsvFile(sessionDataSet, path); + } + sessionDataSet.closeOperationHandle(); + ioTPrinter.println(Constants.EXPORT_COMPLETELY); + } catch (StatementExecutionException + | IoTDBConnectionException + | IOException + | WriteProcessException e) { + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); + } + } + + private void exportToSqlFile(SessionDataSet sessionDataSet, String filePath) + throws IoTDBConnectionException, StatementExecutionException, IOException { + List headers = sessionDataSet.getColumnNames(); + int fileIndex = 0; + String deviceName = null; + boolean writeNull = false; + List seriesList = new ArrayList<>(headers); + if (CollectionUtils.isEmpty(headers) || headers.size() <= 1) { + writeNull = true; + } else { + if (headers.contains("Device")) { + seriesList.remove("Time"); + seriesList.remove("Device"); + } else { + Path path = new Path(seriesList.get(1), true); + deviceName = path.getDeviceString(); + seriesList.remove("Time"); + for (int i = 0; i < seriesList.size(); i++) { + String series = seriesList.get(i); + path = new Path(series, true); + seriesList.set(i, path.getMeasurement()); + } + } + } + boolean hasNext = sessionDataSet.hasNext(); + while (hasNext) { + final String finalFilePath = filePath + "_" + fileIndex + ".sql"; + try (FileWriter writer = new FileWriter(finalFilePath)) { + if (writeNull) { + break; + } + int i = 0; + while (i++ < linesPerFile && hasNext) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + List headersTemp = new ArrayList<>(seriesList); + List timeseries = new ArrayList<>(); + if (headers.contains("Device")) { + deviceName = fields.get(0).toString(); + if (deviceName.startsWith(SYSTEM_DATABASE + ".")) { + continue; + } + for (String header : headersTemp) { + timeseries.add(deviceName + "." + header); + } + } else { + if (headers.get(1).startsWith(SYSTEM_DATABASE + ".")) { + continue; + } + timeseries.addAll(headers); + timeseries.remove(0); + } + String sqlMiddle = + Boolean.TRUE.equals(aligned) + ? " ALIGNED VALUES (" + rowRecord.getTimestamp() + "," + : " VALUES (" + rowRecord.getTimestamp() + ","; + List values = new ArrayList<>(); + if (headers.contains("Device")) { + fields.remove(0); + } + for (int index = 0; index < fields.size(); index++) { + RowRecord next = + session + .executeQueryStatement("SHOW TIMESERIES " + timeseries.get(index), timeout) + .next(); + if (ObjectUtils.isNotEmpty(next)) { + List timeseriesList = next.getFields(); + String value = fields.get(index).toString(); + if (value.equals("null")) { + headersTemp.remove(seriesList.get(index)); + continue; + } + if ("TEXT".equalsIgnoreCase(timeseriesList.get(3).getStringValue())) { + values.add("\"" + value + "\""); + } else { + values.add(value); + } + } else { + headersTemp.remove(seriesList.get(index)); + } + } + if (CollectionUtils.isNotEmpty(headersTemp)) { + writer.write( + "INSERT INTO " + + deviceName + + "(TIMESTAMP," + + String.join(",", headersTemp) + + ")" + + sqlMiddle + + String.join(",", values) + + ");\n"); + } + hasNext = sessionDataSet.hasNext(); + if (!hasNext) { + break; + } + } + writer.flush(); + } + // 如果没有更多数据，退出循环 + if (!hasNext) { + break; + } + fileIndex++; + } + } + + private static Boolean exportToTsFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, + IoTDBConnectionException, + StatementExecutionException, + WriteProcessException { + List columnNames = sessionDataSet.getColumnNames(); + List columnTypes = sessionDataSet.getColumnTypes(); + File f = FSFactoryProducer.getFSFactory().getFile(filePath); + if (f.exists()) { + Files.delete(f.toPath()); + } + boolean isEmpty = false; + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); + + collectSchemas( + columnNames, columnTypes, deviceSchemaMap, alignedDevices, deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); + + if (!tabletList.isEmpty()) { + writeWithTablets( + sessionDataSet, tabletList, alignedDevices, tsFileWriter, deviceColumnIndices); + tsFileWriter.flush(); + } else { + isEmpty = true; + } + } + if (isEmpty) { + ioTPrinter.println("!!!Warning:Tablet is empty,no data can be exported."); + return false; + } + return true; + } + + private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, IoTDBConnectionException, StatementExecutionException { + List headers = sessionDataSet.getColumnNames(); + int fileIndex = 0; + boolean hasNext = sessionDataSet.hasNext(); + while (hasNext) { + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; + CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); + csvPrinterWrapper.printRecord(headers); + int i = 0; + while (i++ < linesPerFile && hasNext) { + RowRecord rowRecord = sessionDataSet.next(); + if (rowRecord.getTimestamp() != 0) { + csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); + } + rowRecord + .getFields() + .forEach( + field -> { + String fieldStringValue = field.getStringValue(); + if (!"null".equals(field.getStringValue())) { + if ((field.getDataType() == TSDataType.TEXT + || field.getDataType() == TSDataType.STRING) + && !fieldStringValue.startsWith("root.")) { + fieldStringValue = "\"" + fieldStringValue + "\""; + } + csvPrinterWrapper.print(fieldStringValue); + } else { + csvPrinterWrapper.print(""); + } + }); + csvPrinterWrapper.println(); + hasNext = sessionDataSet.hasNext(); + if (!hasNext) { + break; + } + } + csvPrinterWrapper.flush(); + if (!hasNext) { + break; + } + fileIndex++; + } + } + + private static void writeWithTablets( + SessionDataSet sessionDataSet, + List tabletList, + Set alignedDevices, + TsFileWriter tsFileWriter, + Map> deviceColumnIndices) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + + for (Tablet tablet : tabletList) { + String deviceId = tablet.getDeviceId(); + List columnIndices = deviceColumnIndices.get(deviceId); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + List schemas = tablet.getSchemas(); + + for (int i = 0, columnIndicesSize = columnIndices.size(); i < columnIndicesSize; i++) { + Integer columnIndex = columnIndices.get(i); + IMeasurementSchema measurementSchema = schemas.get(i); + Object value = fields.get(columnIndex - 1).getObjectValue(measurementSchema.getType()); + tablet.addValue(measurementSchema.getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + tablet.reset(); + } + } + } + + for (Tablet tablet : tabletList) { + if (tablet.getRowSize() != 0) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + } + } + } + + private static void writeToTsFile( + Set deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) + throws IOException, WriteProcessException { + if (deviceFilterSet.contains(tablet.getDeviceId())) { + tsFileWriter.writeAligned(tablet); + } else { + tsFileWriter.writeTree(tablet); + } + } + + private static List constructTablets( + Map> deviceSchemaMap, + Set alignedDevices, + TsFileWriter tsFileWriter) + throws WriteProcessException { + List tabletList = new ArrayList<>(deviceSchemaMap.size()); + for (Map.Entry> stringListEntry : deviceSchemaMap.entrySet()) { + String deviceId = stringListEntry.getKey(); + List schemaList = stringListEntry.getValue(); + Tablet tablet = new Tablet(deviceId, schemaList); + tablet.initBitMaps(); + Path path = new Path(tablet.getDeviceId()); + if (alignedDevices.contains(tablet.getDeviceId())) { + tsFileWriter.registerAlignedTimeseries(path, schemaList); + } else { + tsFileWriter.registerTimeseries(path, schemaList); + } + tabletList.add(tablet); + } + return tabletList; + } + + private static void collectSchemas( + List columnNames, + List columnTypes, + Map> deviceSchemaMap, + Set alignedDevices, + Map> deviceColumnIndices) + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < columnNames.size(); i++) { + String column = columnNames.get(i); + if (!column.startsWith("root.")) { + continue; + } + TSDataType tsDataType = getType(columnTypes.get(i)); + Path path = new Path(column, true); + String deviceId = path.getDeviceString(); + // query whether the device is aligned or not + try (SessionDataSet deviceDataSet = + session.executeQueryStatement("show devices " + deviceId, timeout)) { + List deviceList = deviceDataSet.next().getFields(); + if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { + alignedDevices.add(deviceId); + } + } + + // query timeseries metadata + MeasurementSchema measurementSchema = + new MeasurementSchema(path.getMeasurement(), tsDataType); + List seriesList = + session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); + measurementSchema.setEncoding(TSEncoding.valueOf(seriesList.get(4).getStringValue())); + measurementSchema.setCompressionType( + CompressionType.valueOf(seriesList.get(5).getStringValue())); + + deviceSchemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); + deviceColumnIndices.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(i); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java new file mode 100644 index 0000000000000..a02236ce0f4cb --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java @@ -0,0 +1,606 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.utils.NodeUrlUtils; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.ImportTsFileOperation; +import org.apache.iotdb.tool.common.OptionsUtil; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.TSDataType; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +public class ImportData extends AbstractDataTool { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static Session session; + + public static void main(String[] args) throws IoTDBConnectionException { + Options helpOptions = OptionsUtil.createHelpOptions(); + Options tsFileOptions = OptionsUtil.createImportTsFileOptions(); + Options csvOptions = OptionsUtil.createImportCsvOptions(); + Options sqlOptions = OptionsUtil.createImportSqlOptions(); + HelpFormatter hf = new HelpFormatter(); + hf.setOptionComparator(null); + hf.setWidth(Constants.MAX_HELP_CONSOLE_WIDTH); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); + + if (args == null || args.length == 0) { + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + try { + commandLine = parser.parse(helpOptions, args, true); + } catch (org.apache.commons.cli.ParseException e) { + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + final List argList = Arrays.asList(args); + int helpIndex = argList.indexOf(Constants.MINUS + Constants.HELP_ARGS); + int sql_dialect = argList.indexOf(Constants.MINUS + Constants.SQL_DIALECT_ARGS); // -sql_dialect + if (sql_dialect >= 0 + && !Constants.SQL_DIALECT_VALUE_TREE.equalsIgnoreCase(argList.get(sql_dialect + 1))) { + final String sqlDialectValue = argList.get(sql_dialect + 1); + if (Constants.SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) { + sqlDialectTree = false; + tsFileOptions = OptionsUtil.createTableImportTsFileOptions(); + csvOptions = OptionsUtil.createTableImportCsvOptions(); + sqlOptions = OptionsUtil.createTableImportSqlOptions(); + } else { + ioTPrinter.println(String.format("sql_dialect %s is not support", sqlDialectValue)); + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + } + int ftIndex = argList.indexOf(Constants.MINUS + Constants.FILE_TYPE_ARGS); // -ft + if (ftIndex < 0) { + ftIndex = argList.indexOf(Constants.MINUS + Constants.FILE_TYPE_NAME); // -file_type + } + if (helpIndex >= 0) { + fileType = argList.get(helpIndex + 1); + if (StringUtils.isNotBlank(fileType)) { + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, Constants.IMPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); + } else if (Constants.CSV_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, Constants.IMPORT_CLI_PREFIX, hf, null, csvOptions, null, false); + } else if (Constants.SQL_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, Constants.IMPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); + } else { + ioTPrinter.println(String.format("File type %s is not support", fileType)); + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + } + } else { + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + } + System.exit(Constants.CODE_ERROR); + } else if (ftIndex >= 0) { + fileType = argList.get(ftIndex + 1); + if (StringUtils.isNotBlank(fileType)) { + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(tsFileOptions, args); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions( + null, Constants.IMPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); + System.exit(Constants.CODE_ERROR); + } + } else if (Constants.CSV_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(csvOptions, args); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, Constants.IMPORT_CLI_PREFIX, hf, null, csvOptions, null, false); + System.exit(Constants.CODE_ERROR); + } + } else if (Constants.SQL_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(sqlOptions, args); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, Constants.IMPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); + System.exit(Constants.CODE_ERROR); + } + } else { + ioTPrinter.println(String.format("File type %s is not support", fileType)); + printHelpOptions( + Constants.IMPORT_CLI_HEAD, + Constants.IMPORT_CLI_PREFIX, + hf, + tsFileOptions, + csvOptions, + sqlOptions, + true); + System.exit(Constants.CODE_ERROR); + } + } else { + ioTPrinter.println( + String.format( + "Invalid args: Required values for option '%s' not provided", + Constants.FILE_TYPE_NAME)); + System.exit(Constants.CODE_ERROR); + } + } else { + ioTPrinter.println( + String.format( + "Invalid args: Required values for option '%s' not provided", + Constants.FILE_TYPE_NAME)); + System.exit(Constants.CODE_ERROR); + } + + try { + parseBasicParams(commandLine); + String filename = commandLine.getOptionValue(Constants.FILE_ARGS); + if (filename == null) { + ioTPrinter.println(Constants.IMPORT_CLI_HEAD); + printHelpOptions( + null, Constants.IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(Constants.CODE_ERROR); + } + parseSpecialParams(commandLine); + } catch (ArgsErrorException e) { + ioTPrinter.println("Args error: " + e.getMessage()); + System.exit(Constants.CODE_ERROR); + } catch (Exception e) { + ioTPrinter.println("Encounter an error, because: " + e.getMessage()); + System.exit(Constants.CODE_ERROR); + } + int resultCode = importFromTargetPathAsync(); + System.exit(resultCode); + } + + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { + timeZoneID = commandLine.getOptionValue(Constants.TIME_ZONE_ARGS); + targetPath = commandLine.getOptionValue(Constants.FILE_ARGS); + if (commandLine.getOptionValue(Constants.BATCH_POINT_SIZE_ARGS) != null) { + batchPointSize = + Integer.parseInt(commandLine.getOptionValue(Constants.BATCH_POINT_SIZE_ARGS)); + } + if (commandLine.getOptionValue(Constants.FAIL_DIR_ARGS) != null) { + failedFileDirectory = commandLine.getOptionValue(Constants.FAIL_DIR_ARGS); + File file = new File(failedFileDirectory); + if (!file.isDirectory()) { + file.mkdir(); + failedFileDirectory = file.getAbsolutePath() + File.separator; + } else if (!failedFileDirectory.endsWith("/") && !failedFileDirectory.endsWith("\\")) { + failedFileDirectory += File.separator; + } + } + if (commandLine.getOptionValue(Constants.ALIGNED_ARGS) != null) { + aligned = Boolean.valueOf(commandLine.getOptionValue(Constants.ALIGNED_ARGS)); + } + if (commandLine.getOptionValue(Constants.THREAD_NUM_ARGS) != null) { + threadNum = Integer.parseInt(commandLine.getOptionValue(Constants.THREAD_NUM_ARGS)); + if (threadNum <= 0) { + ioTPrinter.println( + String.format( + "error: Invalid thread number '%s'. Please set a positive integer.", threadNum)); + System.exit(Constants.CODE_ERROR); + } + } + if (commandLine.getOptionValue(Constants.TIMESTAMP_PRECISION_ARGS) != null) { + timestampPrecision = commandLine.getOptionValue(Constants.TIMESTAMP_PRECISION_ARGS); + } + final String[] opTypeInferValues = commandLine.getOptionValues(Constants.TYPE_INFER_ARGS); + if (opTypeInferValues != null && opTypeInferValues.length > 0) { + for (String opTypeInferValue : opTypeInferValues) { + if (opTypeInferValue.contains("=")) { + final String[] typeInfoExpressionArr = opTypeInferValue.split("="); + final String key = typeInfoExpressionArr[0]; + final String value = typeInfoExpressionArr[1]; + applyTypeInferArgs(key, value); + } + } + } + if (commandLine.getOptionValue(Constants.LINES_PER_FAILED_FILE_ARGS) != null) { + linesPerFailedFile = + Integer.parseInt(commandLine.getOptionValue(Constants.LINES_PER_FAILED_FILE_ARGS)); + } + if (commandLine.getOptionValue(Constants.DB_ARGS) != null) { + database = commandLine.getOptionValue(Constants.DB_ARGS); + } + if (commandLine.getOptionValue(Constants.TABLE_ARGS) != null) { + table = commandLine.getOptionValue(Constants.TABLE_ARGS); + } + if (commandLine.getOptionValue(Constants.START_TIME_ARGS) != null) { + startTime = commandLine.getOptionValue(Constants.START_TIME_ARGS); + } + if (commandLine.getOptionValue(Constants.END_TIME_ARGS) != null) { + endTime = commandLine.getOptionValue(Constants.END_TIME_ARGS); + } + try { + isRemoteLoad = !NodeUrlUtils.containsLocalAddress(Collections.singletonList(host)); + if (!sqlDialectTree && isRemoteLoad && Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + ioTPrinter.println( + "host: " + host + " is remote load,only local load is supported in table model"); + } + } catch (UnknownHostException e) { + ioTPrinter.println( + "Unknown host: " + host + ". Exception: " + e.getMessage() + ". Will use local load."); + } + final String os = commandLine.getOptionValue(Constants.ON_SUCCESS_ARGS); + final String onSuccess = StringUtils.isNotBlank(os) ? os.trim().toLowerCase() : null; + final String of = commandLine.getOptionValue(Constants.ON_FAIL_ARGS); + final String onFail = StringUtils.isNotBlank(of) ? of.trim().toLowerCase() : null; + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType) + && (!ImportTsFileOperation.isValidOperation(onSuccess) + || !ImportTsFileOperation.isValidOperation(onFail))) { + ioTPrinter.println("Args error: os/of must be one of none, mv, cp, delete"); + System.exit(Constants.CODE_ERROR); + } + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + boolean isSuccessDirEqualsSourceDir = false; + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onSuccess) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onSuccess)) { + File dir = createSuccessDir(commandLine); + isSuccessDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); + } + + boolean isFailDirEqualsSourceDir = false; + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onFail) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onFail)) { + File dir = createFailDir(commandLine); + isFailDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); + } + successOperation = ImportTsFileOperation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = ImportTsFileOperation.getOperation(onFail, isFailDirEqualsSourceDir); + } + if (!sqlDialectTree + && Constants.CSV_SUFFIXS.equalsIgnoreCase(fileType) + && StringUtils.isBlank(table)) { + ioTPrinter.println("Invalid args: Required values for option table not provided."); + System.exit(Constants.CODE_ERROR); + } + } + + public static boolean isFileStoreEquals(String pathString, File dir) { + try { + return Objects.equals( + Files.getFileStore(Paths.get(pathString)), Files.getFileStore(dir.toPath())); + } catch (IOException e) { + ioTPrinter.println("IOException when checking file store: " + e.getMessage()); + return false; + } + } + + public static File createSuccessDir(CommandLine commandLine) { + if (commandLine.getOptionValue(Constants.SUCCESS_DIR_ARGS) != null) { + successDir = commandLine.getOptionValue(Constants.SUCCESS_DIR_ARGS); + } + File file = new File(successDir); + if (!file.isDirectory()) { + if (!file.mkdirs()) { + ioTPrinter.println( + String.format("Failed to create %s %s", Constants.SUCCESS_DIR_NAME, successDir)); + System.exit(Constants.CODE_ERROR); + } + } + return file; + } + + public static File createFailDir(CommandLine commandLine) { + if (commandLine.getOptionValue(Constants.FAIL_DIR_ARGS) != null) { + failDir = commandLine.getOptionValue(Constants.FAIL_DIR_ARGS); + } + File file = new File(failDir); + if (!file.isDirectory()) { + if (!file.mkdirs()) { + ioTPrinter.println( + String.format("Failed to create %s %s", Constants.FAIL_DIR_NAME, failDir)); + System.exit(Constants.CODE_ERROR); + } + } + return file; + } + + private static void applyTypeInferArgs(String key, String value) throws ArgsErrorException { + if (!Constants.TYPE_INFER_KEY_DICT.containsKey(key)) { + throw new ArgsErrorException("Unknown type infer key: " + key); + } + if (!Constants.TYPE_INFER_VALUE_DICT.containsKey(value)) { + throw new ArgsErrorException("Unknown type infer value: " + value); + } + if (key.equals(Constants.DATATYPE_NAN) + && !(value.equals(Constants.DATATYPE_FLOAT) + || value.equals(Constants.DATATYPE_DOUBLE) + || value.equals(Constants.DATATYPE_TEXT) + || value.equals(Constants.DATATYPE_STRING))) { + throw new ArgsErrorException("NaN can not convert to " + value); + } + if (key.equals(Constants.DATATYPE_BOOLEAN) + && !(value.equals(Constants.DATATYPE_BOOLEAN) + || value.equals(Constants.DATATYPE_TEXT) + || value.equals(Constants.DATATYPE_STRING))) { + throw new ArgsErrorException("Boolean can not convert to " + value); + } + if (key.equals(Constants.DATATYPE_DATE) + && !(value.equals(Constants.DATATYPE_DATE) + || value.equals(Constants.DATATYPE_TEXT) + || value.equals(Constants.DATATYPE_STRING))) { + throw new ArgsErrorException("Date can not convert to " + value); + } + if (key.equals(Constants.DATATYPE_TIMESTAMP) + && !(value.equals(Constants.DATATYPE_TIMESTAMP) + || value.equals(Constants.DATATYPE_TEXT) + || value.equals(Constants.DATATYPE_STRING) + || value.equals(Constants.DATATYPE_DOUBLE) + || value.equals(Constants.DATATYPE_LONG))) { + throw new ArgsErrorException("Timestamp can not convert to " + value); + } + if (key.equals(Constants.DATATYPE_BLOB) && !(value.equals(Constants.DATATYPE_BLOB))) { + throw new ArgsErrorException("Blob can not convert to " + value); + } + final TSDataType srcType = Constants.TYPE_INFER_VALUE_DICT.get(key); + final TSDataType dstType = Constants.TYPE_INFER_VALUE_DICT.get(value); + if (dstType.getType() < srcType.getType()) { + throw new ArgsErrorException(key + " can not convert to " + value); + } + Constants.TYPE_INFER_KEY_DICT.put(key, Constants.TYPE_INFER_VALUE_DICT.get(value)); + } + + private static int importFromTargetPathAsync() { + try { + AbstractImportData importData; + if (sqlDialectTree) { + importData = new ImportDataTree(); + } else { + importData = new ImportDataTable(); + } + importData.init(); + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(Constants.CODE_ERROR); + } + AbstractImportData.init(importData); + return Constants.CODE_OK; + } catch (InterruptedException e) { + ioTPrinter.println(String.format("Import tsfile fail: %s", e.getMessage())); + Thread.currentThread().interrupt(); + return Constants.CODE_ERROR; + } catch (Exception e) { + ioTPrinter.println(String.format("Import tsfile fail: %s", e.getMessage())); + return Constants.CODE_ERROR; + } + } + + /** + * Specifying a CSV file or a directory including CSV files that you want to import. This method + * can be offered to console cli to implement importing CSV file by command. Available for Cli + * import + * + * @param host + * @param port + * @param username + * @param password + * @param targetPath a CSV file or a directory including CSV files + * @param timeZone + * @return the status code + * @throws IoTDBConnectionException + */ + @SuppressWarnings({"squid:S2093"}) // ignore try-with-resources + public static int importFromTargetPath( + String host, int port, String username, String password, String targetPath, String timeZone) + throws IoTDBConnectionException { + try { + session = new Session(host, port, username, password, false); + session.open(false); + timeZoneID = timeZone; + setTimeZone(); + + File file = new File(targetPath); + if (file.isFile()) { + if (file.getName().endsWith(Constants.SQL_SUFFIXS)) { + importFromSqlFile(session, file); + } else { + importFromSingleFile(session, file); + } + } else if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files == null) { + return Constants.CODE_OK; + } + + for (File subFile : files) { + if (subFile.isFile()) { + if (subFile.getName().endsWith(Constants.SQL_SUFFIXS)) { + importFromSqlFile(session, subFile); + } else { + importFromSingleFile(session, subFile); + } + } + } + } else { + ioTPrinter.println("File not found!"); + return Constants.CODE_ERROR; + } + } catch (IoTDBConnectionException | StatementExecutionException e) { + ioTPrinter.println("Encounter an error when connecting to server, because " + e.getMessage()); + return Constants.CODE_ERROR; + } finally { + if (session != null) { + session.close(); + } + } + return Constants.CODE_OK; + } + + private static void setTimeZone() throws IoTDBConnectionException, StatementExecutionException { + if (timeZoneID != null) { + session.setTimeZone(timeZoneID); + } + zoneId = ZoneId.of(session.getTimeZone()); + } + + /** + * import the CSV file and load headers and records. Available for Cli import + * + * @param file the File object of the CSV file that you want to import. + */ + private static void importFromSingleFile(Session session, File file) { + if (file.getName().endsWith(Constants.CSV_SUFFIXS) + || file.getName().endsWith(Constants.TXT_SUFFIXS)) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println("Empty file!"); + return; + } + if (!timeColumn.equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { + ioTPrinter.println("The first field of header must be `Time`!"); + return; + } + String failedFilePath = null; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + if (!deviceColumn.equalsIgnoreCase(headerNames.get(1))) { + writeDataAlignedByTime(session, headerNames, records, failedFilePath); + } else { + writeDataAlignedByDevice(session, headerNames, records, failedFilePath); + } + } catch (IOException | IllegalPathException e) { + ioTPrinter.println("CSV file read exception because: " + e.getMessage()); + } + } else { + ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); + } + } + + /** + * import the SQL file and load headers and records. Available for Cli import + * + * @param session + * @param file + */ + @SuppressWarnings("java:S2259") + private static void importFromSqlFile(Session session, File file) { + ArrayList> failedRecords = new ArrayList<>(); + String failedFilePath = null; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + String sql; + while ((sql = br.readLine()) != null) { + try { + session.executeNonQueryStatement(sql); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Arrays.asList(sql)); + } + } + ioTPrinter.println(file.getName() + " Import completely!"); + } catch (IOException e) { + ioTPrinter.println("SQL file read exception because: " + e.getMessage()); + } + if (!failedRecords.isEmpty()) { + FileWriter writer = null; + try { + writer = new FileWriter(failedFilePath); + for (List failedRecord : failedRecords) { + writer.write(failedRecord.get(0).toString() + "\n"); + } + } catch (IOException e) { + ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); + } finally { + if (ObjectUtils.isNotEmpty(writer)) { + try { + writer.flush(); + writer.close(); + } catch (IOException e) { + } + } + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java new file mode 100644 index 0000000000000..8a5f5ce6a2385 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import java.io.File; +import java.util.concurrent.LinkedBlockingQueue; + +public class ImportDataScanTool { + + private static final LinkedBlockingQueue dataQueue = new LinkedBlockingQueue<>(); + private static String sourceFullPath; + + public static void traverseAndCollectFiles() throws InterruptedException { + traverseAndCollectFilesBySourceFullPath(new File(sourceFullPath)); + } + + private static void traverseAndCollectFilesBySourceFullPath(final File file) + throws InterruptedException { + if (file.isFile()) { + putToQueue(file.getAbsolutePath()); + } else if (file.isDirectory()) { + final File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + traverseAndCollectFilesBySourceFullPath(f); + } + } + } + } + + public static String pollFromQueue() { + return ImportDataScanTool.dataQueue.poll(); + } + + public static void putToQueue(final String filePath) throws InterruptedException { + ImportDataScanTool.dataQueue.put(filePath); + } + + public static void setSourceFullPath(final String sourceFullPath) { + ImportDataScanTool.sourceFullPath = sourceFullPath; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java new file mode 100644 index 0000000000000..1b3e2689e9cc1 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.tsfile.ImportTsFileScanTool; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ImportDataTable extends AbstractImportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static ITableSessionPool sessionPool; + private static Map dataTypes = new HashMap<>(); + private static Map columnCategory = new HashMap<>(); + + public void init() throws InterruptedException { + sessionPool = + new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(host + ":" + port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .database(database) + .build(); + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(Constants.CODE_ERROR); + } + // checkDataBase + SessionDataSet sessionDataSet = null; + try (ITableSession session = sessionPool.getSession()) { + List databases = new ArrayList<>(); + sessionDataSet = session.executeQueryStatement("show databases"); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + databases.add(rowRecord.getField(0).getStringValue()); + } + if (!databases.contains(database)) { + ioTPrinter.println(String.format(Constants.TARGET_DATABASE_NOT_EXIST_MSG, database)); + System.exit(1); + } + if (Constants.CSV_SUFFIXS.equals(fileType)) { + if (StringUtils.isNotBlank(table)) { + sessionDataSet = session.executeQueryStatement("show tables"); + List tables = new ArrayList<>(); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + tables.add(rowRecord.getField(0).getStringValue()); + } + if (!tables.contains(table)) { + ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, table)); + System.exit(1); + } + sessionDataSet = session.executeQueryStatement("describe " + table); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + final String columnName = rowRecord.getField(0).getStringValue(); + final String category = rowRecord.getField(2).getStringValue(); + if (!timeColumn.equalsIgnoreCase(category)) { + dataTypes.put(columnName, getType(rowRecord.getField(1).getStringValue())); + columnCategory.put(columnName, getColumnCategory(category)); + } + } + } else { + ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, null)); + System.exit(1); + } + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (Exception e) { + } + } + } + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + ImportTsFileScanTool.setSourceFullPath(targetPath); + ImportTsFileScanTool.traverseAndCollectFiles(); + ImportTsFileScanTool.addNoResourceOrModsToQueue(); + } else { + ImportDataScanTool.setSourceFullPath(targetPath); + ImportDataScanTool.traverseAndCollectFiles(); + } + } + + @Override + protected Runnable getAsyncImportRunnable() { + return new ImportDataTable(); // 返回子类1的Runnable对象 + } + + protected static void processSuccessFile() { + loadFileSuccessfulNum.increment(); + } + + @SuppressWarnings("java:S2259") + protected void importFromSqlFile(File file) { + ArrayList> failedRecords = new ArrayList<>(); + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + String sql; + while ((sql = br.readLine()) != null) { + try (ITableSession session = sessionPool.getSession()) { + sql = sql.replace(";", ""); + session.executeNonQueryStatement(sql); + } catch (IoTDBConnectionException | StatementExecutionException e) { + ioTPrinter.println(e.getMessage()); + failedRecords.add(Collections.singletonList(sql)); + } + } + processSuccessFile(); + } catch (IOException e) { + ioTPrinter.println("SQL file read exception because: " + e.getMessage()); + } + if (!failedRecords.isEmpty()) { + FileWriter writer = null; + try { + writer = new FileWriter(failedFilePath); + for (List failedRecord : failedRecords) { + writer.write(failedRecord.get(0).toString() + "\n"); + } + } catch (IOException e) { + ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); + } finally { + if (ObjectUtils.isNotEmpty(writer)) { + try { + writer.flush(); + writer.close(); + } catch (IOException e) { + ; + } + } + } + } + } + + protected void importFromTsFile(File file) { + final String sql = "load '" + file + "'"; + try (ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement(sql); + processSuccessFile(file.getPath()); + } catch (final Exception e) { + processFailFile(file.getPath(), e); + } + } + + protected void importFromCsvFile(File file) { + if (file.getName().endsWith(Constants.CSV_SUFFIXS) + || file.getName().endsWith(Constants.TXT_SUFFIXS)) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println("Empty file!"); + return; + } + if (!timeColumn.toLowerCase().equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { + ioTPrinter.println("The first field of header must be `time`!"); + return; + } + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + writeData(headerNames, records.collect(Collectors.toList()), failedFilePath); + processSuccessFile(); + } catch (IOException e) { + ioTPrinter.println("CSV file read exception because: " + e.getMessage()); + } + } else { + ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); + } + } + + protected void writeData( + List headerNames, List records, String failedFilePath) { + Map headerTypeMap = new HashMap<>(); + Map headerNameMap = new HashMap<>(); + parseHeaders(headerNames, headerTypeMap, headerNameMap); + queryType(headerTypeMap); + List> failedRecords = new ArrayList<>(); + List schemas = new ArrayList<>(); + headerNames.forEach( + t -> { + if (dataTypes.containsKey(t)) { + schemas.add(new MeasurementSchema(t, dataTypes.get(t))); + } + }); + List headNames = new LinkedList<>(dataTypes.keySet()); + List columnTypes = new LinkedList<>(dataTypes.values()); + List columnCategorys = new LinkedList<>(columnCategory.values()); + Tablet tablet = new Tablet(table, headNames, columnTypes, columnCategorys, batchPointSize); + for (CSVRecord recordObj : records) { + boolean isFail = false; + final int rowSize = tablet.getRowSize(); + long rowTimeStamp = parseTimestamp(recordObj.get(0)); + for (String headerName : headerNameMap.keySet()) { + String value = recordObj.get(headerNameMap.get(headerName)); + if (!"".equals(value)) { + TSDataType type; + if (timeColumn.equalsIgnoreCase(headerName)) { + tablet.addTimestamp(rowSize, rowTimeStamp); + continue; + } else if (!headerTypeMap.containsKey(headerName)) { + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(headerName, type); + int newIndex = headerNames.indexOf(headerName); + if (newIndex >= columnTypes.size()) { + headNames.add(headerName); + columnTypes.add(type); + columnCategorys.add(ColumnCategory.FIELD); + } else { + headNames.add(headerName); + columnTypes.add(newIndex, type); + columnCategorys.add(newIndex, ColumnCategory.FIELD); + } + writeAndEmptyDataSet(tablet, 3); + tablet = new Tablet(table, headNames, columnTypes, columnCategorys, batchPointSize); + tablet.addTimestamp(rowSize, rowTimeStamp); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), headerName, value); + isFail = true; + } + } + type = headerTypeMap.get(headerName); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail = true; + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), headerName, value, type); + } else { + tablet.addValue(headerName, rowSize, valueTrans); + } + } + } + } + if (tablet.getRowSize() >= batchPointSize) { + writeAndEmptyDataSet(tablet, 3); + tablet.reset(); + } + if (isFail) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + } + if (tablet.getRowSize() > 0) { + writeAndEmptyDataSet(tablet, 3); + } + + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + } + + private static void writeAndEmptyDataSet(Tablet tablet, int retryTime) { + try (ITableSession session = sessionPool.getSession()) { + session.insert(tablet); + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet(tablet, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } + } + + private void parseHeaders( + List headerNames, + Map headerTypeMap, + Map headerNameMap) { + String regex = "(?<=\\()\\S+(?=\\))"; + Pattern pattern = Pattern.compile(regex); + for (String headerName : headerNames) { + headerName = headerName.toLowerCase(); + Matcher matcher = pattern.matcher(headerName); + String type; + String headerNameWithoutType; + if (matcher.find()) { + type = matcher.group(); + headerNameWithoutType = headerName.replace("(" + type + ")", "").replaceAll("\\s+", ""); + headerNameMap.put(headerNameWithoutType, headerName); + headerTypeMap.put(headerNameWithoutType, getType(type)); + } else { + headerNameMap.put(headerName, headerName); + } + } + } + + private void queryType(Map dataType) { + if (MapUtils.isEmpty(dataType)) { + dataType.putAll(dataTypes); + } else { + List noMatch = new ArrayList<>(); + for (String headName : dataType.keySet()) { + if (dataTypes.containsKey(headName) + && !dataTypes.get(headName).equals(dataType.get(headName))) { + noMatch.add(headName); + } + } + if (CollectionUtils.isNotEmpty(noMatch)) { + StringBuilder sb = new StringBuilder(); + for (String match : noMatch) { + sb.append(match) + .append(": rawType(") + .append(dataType.get(match)) + .append("), targetType(") + .append(dataTypes.get(match)) + .append(");"); + } + ioTPrinter.println( + "These columns do not match the column types in the target table：" + sb.toString()); + System.exit(1); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTree.java new file mode 100644 index 0000000000000..ec673f5ff27ec --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTree.java @@ -0,0 +1,501 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.isession.pool.SessionDataSetWrapper; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.tsfile.ImportTsFileScanTool; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ImportDataTree extends AbstractImportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static SessionPool sessionPool; + + public void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException { + sessionPool = + new SessionPool.Builder() + .host(host) + .port(Integer.parseInt(port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .build(); + sessionPool.setEnableQueryRedirection(false); + if (timeZoneID != null) { + sessionPool.setTimeZone(timeZoneID); + zoneId = sessionPool.getZoneId(); + } + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(Constants.CODE_ERROR); + } + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + ImportTsFileScanTool.setSourceFullPath(targetPath); + ImportTsFileScanTool.traverseAndCollectFiles(); + ImportTsFileScanTool.addNoResourceOrModsToQueue(); + } else { + ImportDataScanTool.setSourceFullPath(targetPath); + ImportDataScanTool.traverseAndCollectFiles(); + } + } + + @Override + protected Runnable getAsyncImportRunnable() { + return new ImportDataTree(); // 返回子类1的Runnable对象 + } + + @SuppressWarnings("java:S2259") + protected void importFromSqlFile(File file) { + ArrayList> failedRecords = new ArrayList<>(); + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + String sql; + while ((sql = br.readLine()) != null) { + try { + sessionPool.executeNonQueryStatement(sql); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Collections.singletonList(sql)); + } + } + processSuccessFile(null); + } catch (IOException e) { + ioTPrinter.println("SQL file read exception because: " + e.getMessage()); + } + if (!failedRecords.isEmpty()) { + try (FileWriter writer = new FileWriter(failedFilePath)) { + for (List failedRecord : failedRecords) { + writer.write(failedRecord.get(0).toString() + "\n"); + } + } catch (IOException e) { + ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); + } + } + } + + protected void importFromTsFile(File file) { + final String sql = "load '" + file + "' onSuccess=none "; + try { + sessionPool.executeNonQueryStatement(sql); + processSuccessFile(file.getPath()); + } catch (final Exception e) { + processFailFile(file.getPath(), e); + } + } + + protected void importFromCsvFile(File file) { + if (file.getName().endsWith(Constants.CSV_SUFFIXS) + || file.getName().endsWith(Constants.TXT_SUFFIXS)) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println("Empty file!"); + return; + } + if (!timeColumn.equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { + ioTPrinter.println("The first field of header must be `Time`!"); + return; + } + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + if (!deviceColumn.equalsIgnoreCase(headerNames.get(1))) { + writeDataAlignedByTime(headerNames, records, failedFilePath); + } else { + writeDataAlignedByDevice(headerNames, records, failedFilePath); + } + processSuccessFile(null); + } catch (IOException | IllegalPathException e) { + ioTPrinter.println("CSV file read exception because: " + e.getMessage()); + } + } else { + ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); + } + } + + /** + * if the data is aligned by time, the data will be written by this method. + * + * @param headerNames the header names of CSV file + * @param records the records of CSV file + * @param failedFilePath the directory to save the failed files + */ + @SuppressWarnings("squid:S3776") + protected static void writeDataAlignedByTime( + List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + HashMap> deviceAndMeasurementNames = new HashMap<>(); + HashMap headerTypeMap = new HashMap<>(); + HashMap headerNameMap = new HashMap<>(); + parseHeaders(headerNames, deviceAndMeasurementNames, headerTypeMap, headerNameMap); + + Set devices = deviceAndMeasurementNames.keySet(); + if (headerTypeMap.isEmpty()) { + queryType(devices, headerTypeMap, "Time"); + } + + List deviceIds = new ArrayList<>(); + List times = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + List> typesList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + + AtomicReference hasStarted = new AtomicReference<>(false); + AtomicInteger pointSize = new AtomicInteger(0); + + ArrayList> failedRecords = new ArrayList<>(); + + records.forEach( + recordObj -> { + if (Boolean.FALSE.equals(hasStarted.get())) { + hasStarted.set(true); + } else if (pointSize.get() >= batchPointSize) { + writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + boolean isFail = false; + for (Map.Entry> entry : deviceAndMeasurementNames.entrySet()) { + String deviceId = entry.getKey(); + List measurementNames = entry.getValue(); + ArrayList types = new ArrayList<>(); + ArrayList values = new ArrayList<>(); + ArrayList measurements = new ArrayList<>(); + for (String measurement : measurementNames) { + String header = deviceId + "." + measurement; + String value = recordObj.get(headerNameMap.get(header)); + if (!"".equals(value)) { + TSDataType type; + if (!headerTypeMap.containsKey(header)) { + queryType(header, headerTypeMap); + if (!headerTypeMap.containsKey(header)) { + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(header, type); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), header, value); + isFail = true; + } + } + } + type = headerTypeMap.get(header); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail = true; + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), header, value, type); + } else { + measurements.add(header.replace(deviceId + '.', "")); + types.add(type); + values.add(valueTrans); + pointSize.getAndIncrement(); + } + } + } + } + if (!measurements.isEmpty()) { + times.add(parseTimestamp(recordObj.get(timeColumn))); + deviceIds.add(deviceId); + typesList.add(types); + valuesList.add(values); + measurementsList.add(measurements); + } + } + if (isFail) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + }); + if (!deviceIds.isEmpty()) { + writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + } + + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + protected static void writeDataAlignedByDevice( + List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + HashMap headerTypeMap = new HashMap<>(); + HashMap headerNameMap = new HashMap<>(); + parseHeaders(headerNames, null, headerTypeMap, headerNameMap); + AtomicReference deviceName = new AtomicReference<>(null); + HashSet typeQueriedDevice = new HashSet<>(); + + // the data that interface need + List times = new ArrayList<>(); + List> typesList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + + AtomicInteger pointSize = new AtomicInteger(0); + + List> failedRecords = new ArrayList<>(); + + records.forEach( + recordObj -> { + // only run in first record + if (deviceName.get() == null) { + deviceName.set(recordObj.get(1)); + } else if (!Objects.equals(deviceName.get(), recordObj.get(1))) { + // if device changed + writeAndEmptyDataSet( + deviceName.get(), times, typesList, valuesList, measurementsList, 3); + deviceName.set(recordObj.get(1)); + pointSize.set(0); + } else if (pointSize.get() >= batchPointSize) { + // insert a batch + writeAndEmptyDataSet( + deviceName.get(), times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + // the data of the record + ArrayList types = new ArrayList<>(); + ArrayList values = new ArrayList<>(); + ArrayList measurements = new ArrayList<>(); + + AtomicReference isFail = new AtomicReference<>(false); + + // read data from record + for (Map.Entry headerNameEntry : headerNameMap.entrySet()) { + // headerNameWithoutType is equal to headerName if the CSV column do not have data type. + String headerNameWithoutType = headerNameEntry.getKey(); + String headerName = headerNameEntry.getValue(); + String value = recordObj.get(headerName); + if (!"".equals(value)) { + TSDataType type; + if (!headerTypeMap.containsKey(headerNameWithoutType)) { + // query the data type in iotdb + if (!typeQueriedDevice.contains(deviceName.get())) { + if (headerTypeMap.isEmpty()) { + Set devices = new HashSet<>(); + devices.add(deviceName.get()); + queryType(devices, headerTypeMap, deviceColumn); + } + typeQueriedDevice.add(deviceName.get()); + } + if (!headerTypeMap.containsKey(headerNameWithoutType)) { + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(headerNameWithoutType, type); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), headerNameWithoutType, value); + isFail.set(true); + } + } + } + type = headerTypeMap.get(headerNameWithoutType); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail.set(true); + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), headerNameWithoutType, value, type); + } else { + values.add(valueTrans); + measurements.add(headerNameWithoutType); + types.add(type); + pointSize.getAndIncrement(); + } + } + } + } + if (Boolean.TRUE.equals(isFail.get())) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + if (!measurements.isEmpty()) { + times.add(parseTimestamp(recordObj.get(timeColumn))); + typesList.add(types); + valuesList.add(values); + measurementsList.add(measurements); + } + }); + if (!times.isEmpty()) { + writeAndEmptyDataSet(deviceName.get(), times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + } + + private static void writeAndEmptyDataSet( + String device, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + sessionPool.insertRecordsOfOneDevice( + device, times, measurementsList, typesList, valuesList); + } else { + sessionPool.insertAlignedRecordsOfOneDevice( + device, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet(device, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } finally { + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + private static void writeAndEmptyDataSet( + List deviceIds, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + sessionPool.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); + } else { + sessionPool.insertAlignedRecords(deviceIds, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet( + deviceIds, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } finally { + deviceIds.clear(); + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + private static void queryType( + Set deviceNames, HashMap headerTypeMap, String alignedType) { + for (String deviceName : deviceNames) { + String sql = "show timeseries " + deviceName + ".*"; + try (SessionDataSetWrapper sessionDataSetWrapper = sessionPool.executeQueryStatement(sql)) { + int tsIndex = + sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); + int dtIndex = sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); + while (sessionDataSetWrapper.hasNext()) { + RowRecord rowRecord = sessionDataSetWrapper.next(); + List fields = rowRecord.getFields(); + String timeseries = fields.get(tsIndex).getStringValue(); + String dataType = fields.get(dtIndex).getStringValue(); + if (Objects.equals(alignedType, "Time")) { + headerTypeMap.put(timeseries, getType(dataType)); + } else if (Objects.equals(alignedType, deviceColumn)) { + String[] split = PathUtils.splitPathToDetachedNodes(timeseries); + String measurement = split[split.length - 1]; + headerTypeMap.put(measurement, getType(dataType)); + } + } + } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { + ioTPrinter.println( + "Meet error when query the type of timeseries because " + e.getMessage()); + System.exit(1); + } + } + } + + private static void queryType(String series, HashMap headerTypeMap) { + String sql = "show timeseries " + series; + try (SessionDataSetWrapper sessionDataSetWrapper = sessionPool.executeQueryStatement(sql)) { + int tsIndex = sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); + int dtIndex = sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); + while (sessionDataSetWrapper.hasNext()) { + RowRecord rowRecord = sessionDataSetWrapper.next(); + List fields = rowRecord.getFields(); + String timeseries = fields.get(tsIndex).getStringValue(); + String dataType = fields.get(dtIndex).getStringValue(); + if (Objects.equals(series, timeseries)) { + headerTypeMap.put(timeseries, getType(dataType)); + } + } + } catch (StatementExecutionException | IoTDBConnectionException e) { + ioTPrinter.println("Meet error when query the type of timeseries because " + e.getMessage()); + System.exit(1); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractExportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractExportSchema.java new file mode 100644 index 0000000000000..902198a7ad2a2 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractExportSchema.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +public abstract class AbstractExportSchema extends AbstractSchemaTool { + + public abstract void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException; + + protected abstract void exportSchemaToSqlFile(); + + protected abstract void exportSchemaToCsvFile(String pathPattern, int index); +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractImportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractImportSchema.java new file mode 100644 index 0000000000000..881cc08bdfd3f --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractImportSchema.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.data.ImportDataScanTool; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractImportSchema extends AbstractSchemaTool implements Runnable { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + public abstract void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException; + + @Override + public void run() { + String filePath; + try { + while ((filePath = ImportDataScanTool.pollFromQueue()) != null) { + File file = new File(filePath); + if (!sqlDialectTree && file.getName().endsWith(Constants.SQL_SUFFIXS)) { + importSchemaFromSqlFile(file); + } else if (sqlDialectTree && file.getName().endsWith(Constants.CSV_SUFFIXS)) { + importSchemaFromCsvFile(file); + } else { + ioTPrinter.println( + file.getName() + + " : The file name must end with \"csv\" when sql_dialect tree or \"sql\" when sql_dialect table!"); + } + } + } catch (Exception e) { + ioTPrinter.println("Unexpected error occurred: " + e.getMessage()); + } + } + + protected abstract Runnable getAsyncImportRunnable(); + + protected class ThreadManager { + public void asyncImportSchemaFiles() { + List list = new ArrayList<>(threadNum); + for (int i = 0; i < threadNum; i++) { + Thread thread = new Thread(getAsyncImportRunnable()); + thread.start(); + list.add(thread); + } + list.forEach( + thread -> { + try { + thread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + ioTPrinter.println("ImportData thread join interrupted: " + e.getMessage()); + } + }); + ioTPrinter.println(Constants.IMPORT_COMPLETELY); + } + } + + public static void init(AbstractImportSchema instance) { + instance.new ThreadManager().asyncImportSchemaFiles(); + } + + protected abstract void importSchemaFromSqlFile(File file); + + protected abstract void importSchemaFromCsvFile(File file); + + protected void processSuccessFile() { + loadFileSuccessfulNum.increment(); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java new file mode 100644 index 0000000000000..23a9127a42780 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.LongAdder; + +public abstract class AbstractSchemaTool { + + protected static String host; + protected static String port; + protected static String table; + protected static String database; + protected static String username; + protected static String password; + protected static Session session; + protected static String queryPath; + protected static int threadNum = 8; + protected static String targetPath; + protected static Boolean sqlDialectTree = true; + protected static long timeout = 60000; + protected static String targetDirectory; + protected static Boolean aligned = false; + protected static int linesPerFile = 10000; + protected static String failedFileDirectory = null; + protected static int batchPointSize = Constants.BATCH_POINT_SIZE; + protected static int linesPerFailedFile = Constants.BATCH_POINT_SIZE; + protected static String targetFile = Constants.DUMP_FILE_NAME_DEFAULT; + protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class); + + protected AbstractSchemaTool() {} + + protected static String checkRequiredArg( + String arg, String name, CommandLine commandLine, String defaultValue) + throws ArgsErrorException { + String str = commandLine.getOptionValue(arg); + if (str == null) { + if (StringUtils.isNotBlank(defaultValue)) { + return defaultValue; + } + String msg = String.format("Required values for option '%s' not provided", name); + throw new ArgsErrorException(msg); + } + return str; + } + + protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException { + host = + checkRequiredArg( + Constants.HOST_ARGS, Constants.HOST_NAME, commandLine, Constants.HOST_DEFAULT_VALUE); + port = + checkRequiredArg( + Constants.PORT_ARGS, Constants.PORT_NAME, commandLine, Constants.PORT_DEFAULT_VALUE); + username = + checkRequiredArg( + Constants.USERNAME_ARGS, + Constants.USERNAME_NAME, + commandLine, + Constants.USERNAME_DEFAULT_VALUE); + password = + checkRequiredArg( + Constants.PW_ARGS, Constants.PW_NAME, commandLine, Constants.PW_DEFAULT_VALUE); + } + + /** + * write data to CSV file. + * + * @param records the records of CSV file + * @param filePath the directory to save the file + */ + public static Boolean writeCsvFile(List> records, String filePath) { + try { + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(filePath); + for (List CsvRecord : records) { + csvPrinterWrapper.printRecordLn(CsvRecord); + } + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + return true; + } catch (IOException e) { + ioTPrinter.printException(e); + return false; + } + } + + static class CSVPrinterWrapper { + private final String filePath; + private final CSVFormat csvFormat; + private CSVPrinter csvPrinter; + + public CSVPrinterWrapper(String filePath) { + this.filePath = filePath; + this.csvFormat = + CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setEscape('\\') + .setQuoteMode(QuoteMode.NONE) + .build(); + } + + public void printRecord(final Iterable values) throws IOException { + if (csvPrinter == null) { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } + csvPrinter.printRecord(values); + } + + public void printRecordLn(final Iterable values) throws IOException { + if (csvPrinter == null) { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } + Iterator var2 = values.iterator(); + + while (var2.hasNext()) { + Object value = var2.next(); + csvPrinter.print(value); + } + csvPrinter.println(); + } + + public void print(Object value) { + if (csvPrinter == null) { + try { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } catch (IOException e) { + ioTPrinter.printException(e); + return; + } + } + try { + csvPrinter.print(value); + } catch (IOException e) { + ioTPrinter.printException(e); + } + } + + public void println() throws IOException { + csvPrinter.println(); + } + + public void close() throws IOException { + if (csvPrinter != null) { + csvPrinter.close(); + } + } + + public void flush() throws IOException { + if (csvPrinter != null) { + csvPrinter.flush(); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchema.java new file mode 100644 index 0000000000000..fe6434ff06c17 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchema.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.type.ExitType; +import org.apache.iotdb.cli.utils.CliContext; +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.cli.utils.JlineUtils; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.OptionsUtil; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; +import org.jline.reader.LineReader; + +import java.io.File; + +/** Export Schema CSV file. */ +public class ExportSchema extends AbstractSchemaTool { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + @SuppressWarnings({ + "squid:S3776", + "squid:S2093" + }) // Suppress high Cognitive Complexity warning, ignore try-with-resources + /* main function of export csv tool. */ + public static void main(String[] args) { + Options options = OptionsUtil.createExportSchemaOptions(); + HelpFormatter hf = new HelpFormatter(); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); + hf.setOptionComparator(null); // avoid reordering + hf.setWidth(org.apache.iotdb.tool.common.Constants.MAX_HELP_CONSOLE_WIDTH); + + if (args == null || args.length == 0) { + ioTPrinter.println(Constants.SCHEMA_CLI_CHECK_IN_HEAD); + hf.printHelp(Constants.EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + try { + commandLine = parser.parse(options, args); + } catch (ParseException e) { + ioTPrinter.println(e.getMessage()); + hf.printHelp(Constants.EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + if (commandLine.hasOption(Constants.HELP_ARGS)) { + hf.printHelp(Constants.EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + try { + parseBasicParams(commandLine); + parseSpecialParams(commandLine); + } catch (ArgsErrorException e) { + ioTPrinter.println("Args args: " + e.getMessage()); + ioTPrinter.println("Use -help for more information"); + System.exit(Constants.CODE_ERROR); + } catch (Exception e) { + ioTPrinter.println("Encounter an error, because " + e.getMessage()); + System.exit(Constants.CODE_ERROR); + } + System.exit(exportToTargetPath()); + } + + private static int exportToTargetPath() { + AbstractExportSchema exportSchema; + try { + if (sqlDialectTree) { + exportSchema = new ExportSchemaTree(); + exportSchema.init(); + if (sqlDialectTree && queryPath == null) { + LineReader lineReader = + JlineUtils.getLineReader( + new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), + username, + host, + port); + String pathPattern = + lineReader.readLine(Constants.EXPORT_CLI_PREFIX + "> please input path pattern: "); + ioTPrinter.println(pathPattern); + String[] values = pathPattern.trim().split(";"); + for (int i = 0; i < values.length; i++) { + if (StringUtils.isBlank(values[i])) { + continue; + } else { + exportSchema.exportSchemaToCsvFile(values[i], i); + } + } + } else { + exportSchema.exportSchemaToCsvFile(queryPath, 0); + } + } else { + exportSchema = new ExportSchemaTable(); + exportSchema.init(); + exportSchema.exportSchemaToSqlFile(); + } + ioTPrinter.println(Constants.EXPORT_COMPLETELY); + return Constants.CODE_OK; + } catch (InterruptedException e) { + ioTPrinter.println(String.format("Export schema fail: %s", e.getMessage())); + Thread.currentThread().interrupt(); + return Constants.CODE_ERROR; + } catch (Exception e) { + ioTPrinter.println(String.format("Export schema fail: %s", e.getMessage())); + return Constants.CODE_ERROR; + } + } + + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { + targetDirectory = + checkRequiredArg(Constants.TARGET_DIR_ARGS, Constants.TARGET_DIR_NAME, commandLine, null); + queryPath = commandLine.getOptionValue(Constants.TARGET_PATH_ARGS); + targetFile = commandLine.getOptionValue(Constants.TARGET_FILE_ARGS); + targetFile = + checkRequiredArg( + Constants.TARGET_FILE_ARGS, + Constants.TARGET_FILE_NAME, + commandLine, + Constants.DUMP_FILE_NAME_DEFAULT); + String timeoutString = commandLine.getOptionValue(Constants.TIMEOUT_ARGS); + if (timeoutString != null) { + timeout = Long.parseLong(timeoutString); + } + if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { + targetDirectory += File.separator; + } + if (commandLine.getOptionValue(Constants.LINES_PER_FILE_ARGS) != null) { + linesPerFile = Integer.parseInt(commandLine.getOptionValue(Constants.LINES_PER_FILE_ARGS)); + } + database = commandLine.getOptionValue(Constants.DB_ARGS); + table = commandLine.getOptionValue(Constants.TABLE_ARGS); + String sqlDialectValue = + checkRequiredArg( + Constants.SQL_DIALECT_ARGS, + Constants.SQL_DIALECT_ARGS, + commandLine, + Constants.SQL_DIALECT_VALUE_TREE); + if (Constants.SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) { + sqlDialectTree = false; + if (StringUtils.isBlank(database)) { + ioTPrinter.println( + String.format("The database param is required when sql_dialect is table ")); + System.exit(Constants.CODE_ERROR); + } else if (StringUtils.isNotBlank(database) + && "information_schema".equalsIgnoreCase(database)) { + ioTPrinter.println( + String.format("Does not support exporting system databases %s", database)); + System.exit(Constants.CODE_ERROR); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTable.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTable.java new file mode 100644 index 0000000000000..e80bdeac82e71 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTable.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExportSchemaTable extends AbstractExportSchema { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static ITableSessionPool sessionPool; + private static Map tableCommentList = new HashMap<>(); + + public void init() throws InterruptedException { + sessionPool = + new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(host + ":" + port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .database(database) + .build(); + checkDatabase(); + try { + parseTablesBySelectSchema(String.format(Constants.EXPORT_SCHEMA_TABLES_SELECT, database)); + } catch (StatementExecutionException | IoTDBConnectionException e) { + try { + parseTablesByShowSchema(String.format(Constants.EXPORT_SCHEMA_TABLES_SHOW, database)); + } catch (IoTDBConnectionException | StatementExecutionException e1) { + ioTPrinter.println(Constants.INSERT_SQL_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } + } + if (MapUtils.isEmpty(tableCommentList)) { + ioTPrinter.println(Constants.TARGET_TABLE_EMPTY_MSG); + System.exit(1); + } + } + + private void checkDatabase() { + Set databases = new HashSet<>(); + SessionDataSet sessionDataSet = null; + try (ITableSession session = sessionPool.getSession()) { + sessionDataSet = session.executeQueryStatement(Constants.EXPORT_SCHEMA_TABLES_SHOW_DATABASES); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + databases.add(rowRecord.getField(0).getStringValue()); + } + } catch (Exception e) { + ioTPrinter.println(Constants.INSERT_SQL_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + ; + } + } + } + if (!databases.contains(database)) { + ioTPrinter.println(String.format(Constants.TARGET_DATABASE_NOT_EXIST_MSG, database)); + System.exit(1); + } + } + + private static void parseTablesBySelectSchema(String sql) + throws StatementExecutionException, IoTDBConnectionException { + SessionDataSet sessionDataSet = null; + HashMap tables = new HashMap<>(); + try (ITableSession session = sessionPool.getSession()) { + sessionDataSet = session.executeQueryStatement(sql); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + String comment = rowRecord.getField(4).getStringValue(); + tables.putIfAbsent( + rowRecord.getField(1).getStringValue(), comment.equals("null") ? null : comment); + } + } catch (Exception e) { + throw e; + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + ; + } + } + } + if (StringUtils.isNotBlank(table)) { + if (!tables.containsKey(table)) { + ioTPrinter.println(Constants.TARGET_TABLE_EMPTY_MSG); + System.exit(1); + } + tableCommentList.put(table, tables.get(table)); + } else { + tableCommentList.putAll(tables); + } + } + + private static void parseTablesByShowSchema(String sql) + throws StatementExecutionException, IoTDBConnectionException { + SessionDataSet sessionDataSet = null; + HashMap tables = new HashMap<>(); + try (ITableSession session = sessionPool.getSession()) { + sessionDataSet = session.executeQueryStatement(sql); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + String comment = rowRecord.getField(3).getStringValue(); + tables.putIfAbsent( + rowRecord.getField(0).getStringValue(), comment.equals("null") ? null : comment); + } + } catch (Exception e) { + throw e; + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + ; + } + } + } + if (StringUtils.isNotBlank(table)) { + if (!tables.containsKey(table)) { + ioTPrinter.println(Constants.TARGET_TABLE_EMPTY_MSG); + System.exit(1); + } else { + tableCommentList.put(table, tables.get(table)); + } + } else { + tableCommentList.putAll(tables); + } + } + + @Override + protected void exportSchemaToSqlFile() { + File file = new File(targetDirectory); + if (!file.isDirectory()) { + file.mkdir(); + } + String fileName = targetDirectory + targetFile + "_" + database + ".sql"; + final Iterator iterator = tableCommentList.keySet().iterator(); + while (iterator.hasNext()) { + String tableName = iterator.next(); + String comment = tableCommentList.get(tableName); + SessionDataSet sessionDataSet = null; + try (ITableSession session = sessionPool.getSession()) { + sessionDataSet = + session.executeQueryStatement( + String.format(Constants.EXPORT_SCHEMA_COLUMNS_SELECT, database, tableName)); + exportSchemaBySelect(sessionDataSet, fileName, tableName, comment); + } catch (IoTDBConnectionException | StatementExecutionException | IOException e) { + try (ITableSession session = sessionPool.getSession()) { + sessionDataSet = + session.executeQueryStatement( + String.format(Constants.EXPORT_SCHEMA_COLUMNS_DESC, database, tableName)); + exportSchemaByDesc(sessionDataSet, fileName, tableName, comment); + } catch (IoTDBConnectionException | StatementExecutionException | IOException e1) { + ioTPrinter.println(Constants.COLUMN_SQL_MEET_ERROR_MSG + e.getMessage()); + } + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + ; + } + } + } + } + } + + private void exportSchemaByDesc( + SessionDataSet sessionDataSet, String fileName, String tableName, String tableComment) + throws IoTDBConnectionException, StatementExecutionException, IOException { + String dropSql = "DROP TABLE IF EXISTS " + tableName + ";\n"; + StringBuilder sb = new StringBuilder(dropSql); + sb.append("CREATE TABLE " + tableName + "(\n"); + try (FileWriter writer = new FileWriter(fileName, true)) { + boolean hasNext = sessionDataSet.hasNext(); + while (hasNext) { + RowRecord rowRecord = sessionDataSet.next(); + hasNext = sessionDataSet.hasNext(); + List fields = rowRecord.getFields(); + String columnName = fields.get(0).getStringValue(); + String dataType = fields.get(1).getStringValue(); + String category = fields.get(2).getStringValue(); + String comment = fields.get(4).getStringValue(); + comment = comment.equals("null") ? null : comment; + sb.append("\t" + columnName + " " + dataType + " " + category); + if (ObjectUtils.isNotEmpty(comment)) { + sb.append(" COMMENT '" + comment + "'"); + } + if (hasNext) { + sb.append(","); + } + sb.append("\n"); + } + sb.append(")"); + if (StringUtils.isNotBlank(tableComment)) { + sb.append(" COMMENT '" + tableComment + "'"); + } + sb.append(";\n"); + writer.append(sb.toString()); + writer.flush(); + } + } + + private void exportSchemaBySelect( + SessionDataSet sessionDataSet, String fileName, String tableName, String tableComment) + throws IoTDBConnectionException, StatementExecutionException, IOException { + String dropSql = "DROP TABLE IF EXISTS " + tableName + ";\n"; + StringBuilder sb = new StringBuilder(dropSql); + sb.append("CREATE TABLE " + tableName + "(\n"); + try (FileWriter writer = new FileWriter(fileName, true)) { + boolean hasNext = sessionDataSet.hasNext(); + while (hasNext) { + RowRecord rowRecord = sessionDataSet.next(); + hasNext = sessionDataSet.hasNext(); + List fields = rowRecord.getFields(); + String columnName = fields.get(2).getStringValue(); + String dataType = fields.get(3).getStringValue(); + String category = fields.get(4).getStringValue(); + String comment = fields.get(6).getStringValue(); + comment = comment.equals("null") ? null : comment; + sb.append("\t" + columnName + " " + dataType + " " + category); + if (ObjectUtils.isNotEmpty(comment)) { + sb.append(" COMMENT '" + comment + "'"); + } + if (hasNext) { + sb.append(","); + } + sb.append("\n"); + } + sb.append(")"); + if (StringUtils.isNotBlank(tableComment)) { + sb.append(" COMMENT '" + tableComment + "'"); + } + sb.append(";\n"); + writer.append(sb.toString()); + writer.flush(); + } + } + + @Override + protected void exportSchemaToCsvFile(String pathPattern, int index) { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java new file mode 100644 index 0000000000000..7b8c63a49143b --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; + +public class ExportSchemaTree extends AbstractExportSchema { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static Session session; + + public void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException { + session = new Session(host, Integer.parseInt(port), username, password); + session.open(false); + } + + @Override + protected void exportSchemaToSqlFile() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + protected void exportSchemaToCsvFile(String pathPattern, int index) { + File file = new File(targetDirectory); + if (!file.isDirectory()) { + file.mkdir(); + } + final String path = targetDirectory + targetFile + index; + try { + SessionDataSet sessionDataSet = + session.executeQueryStatement("show timeseries " + pathPattern, timeout); + writeCsvFile(sessionDataSet, path, sessionDataSet.getColumnNames(), linesPerFile); + sessionDataSet.closeOperationHandle(); + ioTPrinter.println(Constants.EXPORT_COMPLETELY); + } catch (StatementExecutionException | IoTDBConnectionException | IOException e) { + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); + } + } + + private static void writeCsvFile( + SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) + throws IOException, IoTDBConnectionException, StatementExecutionException { + int viewTypeIndex = headers.indexOf(Constants.HEADER_VIEW_TYPE); + int timeseriesIndex = headers.indexOf(Constants.HEADER_TIMESERIES); + + int fileIndex = 0; + boolean hasNext = true; + while (hasNext) { + int i = 0; + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); + while (i++ < linesPerFile) { + if (sessionDataSet.hasNext()) { + if (i == 1) { + csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS); + } + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE + ".") + || !fields.get(viewTypeIndex).getStringValue().equals(Constants.BASE_VIEW_TYPE)) { + continue; + } + Constants.HEAD_COLUMNS.forEach( + column -> { + Field field = fields.get(headers.indexOf(column)); + String fieldStringValue = field.getStringValue(); + if (!"null".equals(field.getStringValue())) { + csvPrinterWrapper.print(fieldStringValue); + } else { + csvPrinterWrapper.print(""); + } + }); + csvPrinterWrapper.println(); + } else { + hasNext = false; + break; + } + } + fileIndex++; + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchema.java new file mode 100644 index 0000000000000..21e19055b7306 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchema.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.OptionsUtil; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; + +/** Import Schema CSV file. */ +public class ImportSchema extends AbstractSchemaTool { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + /** + * parse optional params + * + * @param commandLine + */ + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { + targetPath = commandLine.getOptionValue(Constants.FILE_ARGS); + if (commandLine.getOptionValue(Constants.BATCH_POINT_SIZE_ARGS) != null) { + batchPointSize = + Integer.parseInt(commandLine.getOptionValue(Constants.BATCH_POINT_SIZE_ARGS)); + } + if (commandLine.getOptionValue(Constants.FAILED_FILE_ARGS) != null) { + failedFileDirectory = commandLine.getOptionValue(Constants.FAILED_FILE_ARGS); + File file = new File(failedFileDirectory); + if (!file.isDirectory()) { + file.mkdir(); + failedFileDirectory = file.getAbsolutePath() + File.separator; + } else if (!failedFileDirectory.endsWith("/") && !failedFileDirectory.endsWith("\\")) { + failedFileDirectory += File.separator; + } + } + if (commandLine.getOptionValue(Constants.ALIGNED_ARGS) != null) { + aligned = Boolean.valueOf(commandLine.getOptionValue(Constants.ALIGNED_ARGS)); + } + if (commandLine.getOptionValue(Constants.LINES_PER_FAILED_FILE_ARGS) != null) { + linesPerFailedFile = + Integer.parseInt(commandLine.getOptionValue(Constants.LINES_PER_FAILED_FILE_ARGS)); + } + database = checkRequiredArg(Constants.DB_ARGS, Constants.DB_NAME, commandLine, null); + String sqlDialectValue = + checkRequiredArg( + Constants.SQL_DIALECT_ARGS, + Constants.SQL_DIALECT_ARGS, + commandLine, + Constants.SQL_DIALECT_VALUE_TREE); + if (Constants.SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) { + sqlDialectTree = false; + if (StringUtils.isBlank(database)) { + ioTPrinter.println( + String.format("The database param is required when sql_dialect is table ")); + System.exit(Constants.CODE_ERROR); + } else if (StringUtils.isNotBlank(database) + && "information_schema".equalsIgnoreCase(database)) { + ioTPrinter.println( + String.format("Does not support exporting system databases %s", database)); + System.exit(Constants.CODE_ERROR); + } + } + } + + public static void main(String[] args) throws IoTDBConnectionException { + Options options = OptionsUtil.createImportSchemaOptions(); + HelpFormatter hf = new HelpFormatter(); + hf.setOptionComparator(null); + hf.setWidth(Constants.MAX_HELP_CONSOLE_WIDTH); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); + + if (args == null || args.length == 0) { + ioTPrinter.println(Constants.SCHEMA_CLI_CHECK_IN_HEAD); + hf.printHelp(Constants.IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + try { + commandLine = parser.parse(options, args); + } catch (org.apache.commons.cli.ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + hf.printHelp(Constants.IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + if (commandLine.hasOption(Constants.HELP_ARGS)) { + hf.printHelp(Constants.IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + try { + parseBasicParams(commandLine); + String filename = commandLine.getOptionValue(Constants.FILE_ARGS); + if (filename == null) { + hf.printHelp(Constants.IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + parseSpecialParams(commandLine); + } catch (ArgsErrorException e) { + ioTPrinter.println("Args error: " + e.getMessage()); + System.exit(Constants.CODE_ERROR); + } catch (Exception e) { + ioTPrinter.println("Encounter an error, because: " + e.getMessage()); + System.exit(Constants.CODE_ERROR); + } + System.exit(importFromTargetPath()); + } + + /** + * Specifying a CSV file or a directory including CSV files that you want to import. This method + * can be offered to console cli to implement importing CSV file by command. + * + * @return the status code + * @throws IoTDBConnectionException + */ + @SuppressWarnings({"squid:S2093"}) // ignore try-with-resources + private static int importFromTargetPath() { + AbstractImportSchema importSchema; + try { + if (sqlDialectTree) { + importSchema = new ImportSchemaTree(); + } else { + importSchema = new ImportSchemaTable(); + } + importSchema.init(); + AbstractImportSchema.init(importSchema); + return Constants.CODE_OK; + } catch (InterruptedException e) { + ioTPrinter.println(String.format("Import schema fail: %s", e.getMessage())); + Thread.currentThread().interrupt(); + return Constants.CODE_ERROR; + } catch (Exception e) { + ioTPrinter.println(String.format("Import schema fail: %s", e.getMessage())); + return Constants.CODE_ERROR; + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTable.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTable.java new file mode 100644 index 0000000000000..25119e48623e4 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTable.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.data.ImportDataScanTool; + +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.read.common.RowRecord; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ImportSchemaTable extends AbstractImportSchema { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static ITableSessionPool sessionPool; + + public void init() throws InterruptedException { + sessionPool = + new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(host + ":" + port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .database(database) + .build(); + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(Constants.CODE_ERROR); + } + SessionDataSet sessionDataSet = null; + try (ITableSession session = sessionPool.getSession()) { + List databases = new ArrayList<>(); + sessionDataSet = session.executeQueryStatement("show databases"); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + databases.add(rowRecord.getField(0).getStringValue()); + } + if (!databases.contains(database)) { + ioTPrinter.println(String.format(Constants.TARGET_DATABASE_NOT_EXIST_MSG, database)); + System.exit(1); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (Exception e) { + } + } + } + ImportDataScanTool.setSourceFullPath(targetPath); + ImportDataScanTool.traverseAndCollectFiles(); + } + + @Override + protected Runnable getAsyncImportRunnable() { + return new ImportSchemaTable(); + } + + @Override + protected void importSchemaFromSqlFile(File file) { + ArrayList> failedRecords = new ArrayList<>(); + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + StringBuilder sqlBuilder = new StringBuilder(); + String sql; + while ((sql = br.readLine()) != null) { + if (!sql.contains(";")) { + sqlBuilder.append(sql); + } else { + String[] sqls = sql.split(";"); + boolean addBuilder = false; + if (sqls.length > 0) { + if (!sql.endsWith(";")) { + addBuilder = true; + } + for (int i = 0; i < sqls.length; i++) { + sqlBuilder.append(sqls[i]); + if (i == sqls.length - 1 && addBuilder) { + break; + } + String builderString = sqlBuilder.toString(); + try (ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement(builderString); + sqlBuilder = new StringBuilder(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Collections.singletonList(builderString)); + sqlBuilder = new StringBuilder(); + } + } + } else { + String builderString = sqlBuilder.toString(); + try (ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement(builderString); + sqlBuilder = new StringBuilder(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Collections.singletonList(builderString)); + sqlBuilder = new StringBuilder(); + } + } + } + } + String builderString = sqlBuilder.toString(); + if (StringUtils.isNotBlank(builderString)) { + try (ITableSession session = sessionPool.getSession()) { + session.executeNonQueryStatement(sqlBuilder.toString()); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Collections.singletonList(builderString)); + sqlBuilder = new StringBuilder(); + } + } + processSuccessFile(); + } catch (IOException e) { + ioTPrinter.println("SQL file read exception because: " + e.getMessage()); + } + if (!failedRecords.isEmpty()) { + FileWriter writer = null; + try { + writer = new FileWriter(failedFilePath); + for (List failedRecord : failedRecords) { + writer.write(failedRecord.get(0).toString() + ";\n"); + } + } catch (IOException e) { + ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); + } finally { + if (ObjectUtils.isNotEmpty(writer)) { + try { + writer.flush(); + writer.close(); + } catch (IOException e) { + ; + } + } + } + } + } + + @Override + protected void importSchemaFromCsvFile(File file) { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java new file mode 100644 index 0000000000000..e97322876735a --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java @@ -0,0 +1,426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.data.ImportDataScanTool; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; + +public class ImportSchemaTree extends AbstractImportSchema { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static SessionPool sessionPool; + + public void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException { + sessionPool = + new SessionPool.Builder() + .host(host) + .port(Integer.parseInt(port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .build(); + sessionPool.setEnableQueryRedirection(false); + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(Constants.CODE_ERROR); + } + ImportDataScanTool.setSourceFullPath(targetPath); + ImportDataScanTool.traverseAndCollectFiles(); + } + + @Override + protected Runnable getAsyncImportRunnable() { + return new ImportSchemaTree(); + } + + @Override + protected void importSchemaFromSqlFile(File file) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + protected void importSchemaFromCsvFile(File file) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println(file.getName() + " : Empty file!"); + return; + } + if (!checkHeader(headerNames)) { + return; + } + String failedFilePath = null; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + writeScheme(file.getName(), headerNames, records, failedFilePath); + processSuccessFile(); + } catch (IOException | IllegalPathException e) { + ioTPrinter.println(file.getName() + " : CSV file read exception because: " + e.getMessage()); + } + } + + private static CSVParser readCsvFile(String path) throws IOException { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setQuote('`') + .setEscape('\\') + .setIgnoreEmptyLines(true) + .build() + .parse(new InputStreamReader(new FileInputStream(path))); + } + + private static boolean checkHeader(List headerNames) { + if (CollectionUtils.isNotEmpty(headerNames) + && new HashSet<>(headerNames).size() == Constants.HEAD_COLUMNS.size()) { + List strangers = + headerNames.stream() + .filter(t -> !Constants.HEAD_COLUMNS.contains(t)) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(strangers)) { + ioTPrinter.println( + "The header of the CSV file to be imported is illegal. The correct format is \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"); + return false; + } + } + return true; + } + + private static void writeScheme( + String fileName, List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + List paths = new ArrayList<>(); + List dataTypes = new ArrayList<>(); + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + List pathsWithAlias = new ArrayList<>(); + List dataTypesWithAlias = new ArrayList<>(); + List encodingsWithAlias = new ArrayList<>(); + List compressorsWithAlias = new ArrayList<>(); + List measurementAlias = new ArrayList<>(); + + AtomicReference hasStarted = new AtomicReference<>(false); + AtomicInteger pointSize = new AtomicInteger(0); + ArrayList> failedRecords = new ArrayList<>(); + records.forEach( + recordObj -> { + boolean failed = false; + if (!aligned) { + if (Boolean.FALSE.equals(hasStarted.get())) { + hasStarted.set(true); + } else if (pointSize.get() >= batchPointSize) { + try { + if (CollectionUtils.isNotEmpty(paths)) { + writeAndEmptyDataSet( + paths, dataTypes, encodings, compressors, null, null, null, null, 3); + } + } catch (Exception e) { + paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); + } + try { + if (CollectionUtils.isNotEmpty(pathsWithAlias)) { + writeAndEmptyDataSet( + pathsWithAlias, + dataTypesWithAlias, + encodingsWithAlias, + compressorsWithAlias, + null, + null, + null, + measurementAlias, + 3); + } + } catch (Exception e) { + paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); + } + paths.clear(); + dataTypes.clear(); + encodings.clear(); + compressors.clear(); + measurementAlias.clear(); + pointSize.set(0); + } + } else { + paths.clear(); + dataTypes.clear(); + encodings.clear(); + compressors.clear(); + measurementAlias.clear(); + } + String path = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(0))); + String alias = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(1))); + String dataTypeRaw = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(2))); + TSDataType dataType = typeInfer(dataTypeRaw); + String encodingTypeRaw = + recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(3))); + TSEncoding encodingType = encodingInfer(encodingTypeRaw); + String compressionTypeRaw = + recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(4))); + CompressionType compressionType = compressInfer(compressionTypeRaw); + if (StringUtils.isBlank(path) || path.trim().startsWith(SYSTEM_DATABASE)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': illegal path %s", + recordObj.getRecordNumber(), headerNames, path)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else if (ObjectUtils.isEmpty(dataType)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': '%s' unknown dataType %n", + recordObj.getRecordNumber(), path, dataTypeRaw)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else if (ObjectUtils.isEmpty(encodingType)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': '%s' unknown encodingType %n", + recordObj.getRecordNumber(), path, encodingTypeRaw)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else if (ObjectUtils.isEmpty(compressionType)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': '%s' unknown compressionType %n", + recordObj.getRecordNumber(), path, compressionTypeRaw)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else { + if (StringUtils.isBlank(alias)) { + paths.add(path); + dataTypes.add(dataType); + encodings.add(encodingType); + compressors.add(compressionType); + } else { + pathsWithAlias.add(path); + dataTypesWithAlias.add(dataType); + encodingsWithAlias.add(encodingType); + compressorsWithAlias.add(compressionType); + measurementAlias.add(alias); + } + pointSize.getAndIncrement(); + } + if (!failed && aligned) { + String deviceId = path.substring(0, path.lastIndexOf(".")); + paths.add(0, path.substring(deviceId.length() + 1)); + writeAndEmptyDataSetAligned( + deviceId, paths, dataTypes, encodings, compressors, measurementAlias, 3); + } + }); + try { + if (CollectionUtils.isNotEmpty(paths)) { + writeAndEmptyDataSet(paths, dataTypes, encodings, compressors, null, null, null, null, 3); + } + } catch (Exception e) { + paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); + } + try { + if (CollectionUtils.isNotEmpty(pathsWithAlias)) { + writeAndEmptyDataSet( + pathsWithAlias, + dataTypesWithAlias, + encodingsWithAlias, + compressorsWithAlias, + null, + null, + null, + measurementAlias, + 3); + } + } catch (Exception e) { + pathsWithAlias.forEach(t -> failedRecords.add(Collections.singletonList(t))); + } + pointSize.set(0); + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(failedFilePath, failedRecords); + } + if (Boolean.TRUE.equals(hasStarted.get())) { + if (!failedRecords.isEmpty()) { + ioTPrinter.println(fileName + " : Import completely fail!"); + } else { + ioTPrinter.println(fileName + " : Import completely successful!"); + } + } else { + ioTPrinter.println(fileName + " : No records!"); + } + } + + private static void writeAndEmptyDataSet( + List paths, + List dataTypes, + List encodings, + List compressors, + List> propsList, + List> tagsList, + List> attributesList, + List measurementAliasList, + int retryTime) + throws StatementExecutionException { + try { + sessionPool.createMultiTimeseries( + paths, + dataTypes, + encodings, + compressors, + propsList, + tagsList, + attributesList, + measurementAliasList); + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet( + paths, + dataTypes, + encodings, + compressors, + propsList, + tagsList, + attributesList, + measurementAliasList, + --retryTime); + } + } catch (StatementExecutionException e) { + throw e; + } + } + + private static void writeAndEmptyDataSetAligned( + String deviceId, + List measurements, + List dataTypes, + List encodings, + List compressors, + List measurementAliasList, + int retryTime) { + try { + sessionPool.createAlignedTimeseries( + deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSetAligned( + deviceId, + measurements, + dataTypes, + encodings, + compressors, + measurementAliasList, + --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } finally { + deviceId = null; + measurements.clear(); + dataTypes.clear(); + encodings.clear(); + compressors.clear(); + measurementAliasList.clear(); + } + } + + private static TSDataType typeInfer(String typeStr) { + try { + if (StringUtils.isNotBlank(typeStr)) { + return TSDataType.valueOf(typeStr); + } + } catch (IllegalArgumentException e) { + ; + } + return null; + } + + private static CompressionType compressInfer(String compressionType) { + try { + if (StringUtils.isNotBlank(compressionType)) { + return CompressionType.valueOf(compressionType); + } + } catch (IllegalArgumentException e) { + ; + } + return null; + } + + private static TSEncoding encodingInfer(String encodingType) { + try { + if (StringUtils.isNotBlank(encodingType)) { + return TSEncoding.valueOf(encodingType); + } + } catch (IllegalArgumentException e) { + ; + } + return null; + } + + private static void writeFailedLinesFile( + String failedFilePath, ArrayList> failedRecords) { + int fileIndex = 0; + int from = 0; + int failedRecordsSize = failedRecords.size(); + int restFailedRecords = failedRecordsSize; + while (from < failedRecordsSize) { + int step = Math.min(restFailedRecords, linesPerFailedFile); + writeCsvFile(failedRecords.subList(from, from + step), failedFilePath + "_" + fileIndex++); + from += step; + restFailedRecords -= step; + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractTsFileTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/AbstractTsFileTool.java similarity index 97% rename from iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractTsFileTool.java rename to iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/AbstractTsFileTool.java index fe8691ddf8b65..64b9f806d5889 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractTsFileTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/AbstractTsFileTool.java @@ -17,10 +17,11 @@ * under the License. */ -package org.apache.iotdb.tool; +package org.apache.iotdb.tool.tsfile; import org.apache.iotdb.cli.utils.IoTPrinter; import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.session.Session; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; @@ -52,7 +53,7 @@ public abstract class AbstractTsFileTool { protected static Options options; protected static Options helpOptions; - + protected static Session session; protected static String host = "127.0.0.1"; protected static String port = "6667"; protected static String username = "root"; diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java new file mode 100644 index 0000000000000..3d8fe0baf91d6 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.OptionsUtil; +import org.apache.iotdb.tool.tsfile.subscription.AbstractSubscriptionTsFile; +import org.apache.iotdb.tool.tsfile.subscription.CommonParam; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class ExportTsFile { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static CommonParam commonParam = CommonParam.getInstance(); + + public static void main(String[] args) throws Exception { + Logger logger = + (Logger) LoggerFactory.getLogger("org.apache.iotdb.session.subscription.consumer.base"); + logger.setLevel(Level.WARN); + Options options = OptionsUtil.createSubscriptionTsFileOptions(); + parseParams(args, options); + if (StringUtils.isEmpty(commonParam.getPath())) { + commonParam.setSqlDialect(Constants.TABLE_MODEL); + } + AbstractSubscriptionTsFile.setSubscriptionSession(); + String nowFormat = Constants.DATE_FORMAT_VIEW.format(System.currentTimeMillis()); + String topicName = Constants.TOPIC_NAME_PREFIX + nowFormat; + String groupId = Constants.GROUP_NAME_PREFIX + nowFormat; + commonParam.getSubscriptionTsFile().createTopics(topicName); + commonParam.getSubscriptionTsFile().createConsumers(groupId); + commonParam.getSubscriptionTsFile().subscribe(topicName); + ExecutorService executor = Executors.newFixedThreadPool(commonParam.getConsumerCount()); + commonParam.getSubscriptionTsFile().consumerPoll(executor, topicName); + executor.shutdown(); + while (true) { + if (executor.isTerminated()) { + break; + } + } + commonParam.getSubscriptionTsFile().doClean(); + ioTPrinter.println("Export TsFile Count: " + commonParam.getCountFile().get()); + } + + private static void parseParams(String[] args, Options options) { + HelpFormatter hf = new HelpFormatter(); + hf.setOptionComparator(null); + CommandLine cli = null; + CommandLineParser cliParser = new DefaultParser(); + try { + cli = cliParser.parse(options, args); + if (cli.hasOption(Constants.HELP_ARGS) || args.length == 0) { + hf.printHelp(Constants.SUBSCRIPTION_CLI_PREFIX, options, true); + System.exit(0); + } + if (cli.hasOption(Constants.SQL_DIALECT_ARGS)) { + commonParam.setSqlDialect(cli.getOptionValue(Constants.SQL_DIALECT_ARGS)); + } + if (cli.hasOption(Constants.HOST_ARGS)) { + commonParam.setSrcHost(cli.getOptionValue(Constants.HOST_ARGS)); + } + if (cli.hasOption(Constants.PORT_ARGS)) { + commonParam.setSrcPort(Integer.valueOf(cli.getOptionValue(Constants.PORT_ARGS))); + } + if (cli.hasOption(Constants.USERNAME_ARGS)) { + commonParam.setSrcUserName(cli.getOptionValue(Constants.USERNAME_ARGS)); + } + if (cli.hasOption(Constants.PW_ARGS)) { + commonParam.setSrcPassword(cli.getOptionValue(Constants.PW_ARGS)); + } + if (cli.hasOption(Constants.PATH_ARGS)) { + commonParam.setPath(cli.getOptionValue(Constants.PATH_ARGS)); + } + if (cli.hasOption(Constants.DB_ARGS)) { + commonParam.setDatabase(cli.getOptionValue(Constants.DB_ARGS)); + } + if (cli.hasOption(Constants.TABLE_ARGS)) { + commonParam.setTable(cli.getOptionValue(Constants.TABLE_ARGS)); + } + if (cli.hasOption(Constants.TARGET_DIR_ARGS)) { + commonParam.setTargetDir(cli.getOptionValue(Constants.TARGET_DIR_ARGS)); + } + if (cli.hasOption(Constants.START_TIME_ARGS)) { + commonParam.setStartTime(cli.getOptionValue(Constants.START_TIME_ARGS)); + } + if (cli.hasOption(Constants.END_TIME_ARGS)) { + commonParam.setEndTime(cli.getOptionValue(Constants.END_TIME_ARGS)); + } + if (cli.hasOption(Constants.THREAD_NUM_ARGS)) { + commonParam.setConsumerCount( + Integer.valueOf(cli.getOptionValue(Constants.THREAD_NUM_ARGS))); + } + } catch (ParseException e) { + ioTPrinter.println(e.getMessage()); + hf.printHelp(Constants.SUBSCRIPTION_CLI_PREFIX, options, true); + System.exit(Constants.CODE_ERROR); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java new file mode 100644 index 0000000000000..c017ab2b0e039 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.utils.NodeUrlUtils; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.ImportTsFileOperation; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.ParseException; + +import java.io.File; +import java.io.IOException; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class ImportTsFile extends AbstractTsFileTool { + + private static final String SOURCE_ARGS = "s"; + private static final String SOURCE_NAME = "source"; + + private static final String ON_SUCCESS_ARGS = "os"; + private static final String ON_SUCCESS_NAME = "on_success"; + + private static final String SUCCESS_DIR_ARGS = "sd"; + private static final String SUCCESS_DIR_NAME = "success_dir"; + + private static final String FAIL_DIR_ARGS = "fd"; + private static final String FAIL_DIR_NAME = "fail_dir"; + + private static final String ON_FAIL_ARGS = "of"; + private static final String ON_FAIL_NAME = "on_fail"; + + private static final String THREAD_NUM_ARGS = "tn"; + private static final String THREAD_NUM_NAME = "thread_num"; + + protected static final String VERIFY_ARGS = "v"; + protected static final String VERIFY_NAME = "verify"; + + private static final IoTPrinter IOT_PRINTER = new IoTPrinter(System.out); + + private static final String TS_FILE_CLI_PREFIX = "ImportTsFile"; + + protected static String timestampPrecision = "ms"; + private static final String TIMESTAMP_PRECISION_ARGS = "tp"; + private static final String TIMESTAMP_PRECISION_NAME = "timestamp_precision"; + private static final String TIMESTAMP_PRECISION_ARGS_NAME = "timestamp precision (ms/us/ns)"; + + private static String source; + + private static String successDir = "success/"; + private static String failDir = "fail/"; + + private static ImportTsFileOperation successOperation; + private static ImportTsFileOperation failOperation; + + private static int threadNum = 8; + + private static boolean isRemoteLoad = true; + protected static boolean verify = true; + private static SessionPool sessionPool; + + private static void createOptions() { + createBaseOptions(); + + Option opSource = + Option.builder(SOURCE_ARGS) + .longOpt(SOURCE_NAME) + .argName(SOURCE_NAME) + .required() + .hasArg() + .desc( + "The source file or directory path, " + + "which can be a tsfile or a directory containing tsfiles. (required)") + .build(); + options.addOption(opSource); + + Option opOnSuccess = + Option.builder(ON_SUCCESS_ARGS) + .longOpt(ON_SUCCESS_NAME) + .argName(ON_SUCCESS_NAME) + .required() + .hasArg() + .desc( + "When loading tsfile successfully, do operation on tsfile (and its .resource and .mods files), " + + "optional parameters are none, mv, cp, delete. (required)") + .build(); + options.addOption(opOnSuccess); + + Option opOnFail = + Option.builder(ON_FAIL_ARGS) + .longOpt(ON_FAIL_NAME) + .argName(ON_FAIL_NAME) + .required() + .hasArg() + .desc( + "When loading tsfile fail, do operation on tsfile (and its .resource and .mods files), " + + "optional parameters are none, mv, cp, delete. (required)") + .build(); + options.addOption(opOnFail); + + Option opSuccessDir = + Option.builder(SUCCESS_DIR_ARGS) + .longOpt(SUCCESS_DIR_NAME) + .argName(SUCCESS_DIR_NAME) + .hasArg() + .desc("The target folder when 'os' is 'mv' or 'cp'.") + .build(); + options.addOption(opSuccessDir); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc("The target folder when 'of' is 'mv' or 'cp'.") + .build(); + options.addOption(opFailDir); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArgs() + .desc("The number of threads used to import tsfile, default is 8.") + .build(); + options.addOption(opThreadNum); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc("Timestamp precision (ms/us/ns) (optional)") + .build(); + + options.addOption(opTimestampPrecision); + } + + public static void main(String[] args) { + long startTime = System.currentTimeMillis(); + createOptions(); + + final CommandLineParser parser = new DefaultParser(); + + final HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setOptionComparator(null); + helpFormatter.setWidth(MAX_HELP_CONSOLE_WIDTH); + + if (args == null || args.length == 0) { + IOT_PRINTER.println("Too few arguments, please check the following hint."); + helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + + try { + if (parser.parse(helpOptions, args, true).hasOption(HELP_ARGS)) { + helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); + System.exit(CODE_OK); + } + } catch (ParseException e) { + IOT_PRINTER.println("Failed to parse the provided options: " + e.getMessage()); + helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args, true); + } catch (ParseException e) { + IOT_PRINTER.println("Failed to parse the provided options: " + e.getMessage()); + helpFormatter.printHelp(TS_FILE_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + + try { + parseBasicParams(commandLine); + parseSpecialParams(commandLine); + } catch (Exception e) { + IOT_PRINTER.println( + "Encounter an error when parsing the provided options: " + e.getMessage()); + System.exit(CODE_ERROR); + } + + IOT_PRINTER.println(isRemoteLoad ? "Load remotely." : "Load locally."); + final int resultCode = importFromTargetPath(); + + ImportTsFileBase.printResult(startTime); + System.exit(resultCode); + } + + private static void checkTimePrecision() { + String precision = null; + if (!isRemoteLoad) { + try { + precision = sessionPool.getTimestampPrecision(); + } catch (Exception e) { + IOT_PRINTER.println( + "Encounter an error when get IoTDB server timestampPrecision : " + e.getMessage()); + System.exit(CODE_ERROR); + } + if (!timestampPrecision.equalsIgnoreCase(precision)) { + IOT_PRINTER.println( + String.format( + "Encounter an error The time accuracy of the iotdb server is \"%s\", but the client time accuracy is %s", + precision, timestampPrecision)); + System.exit(CODE_ERROR); + } + } + } + + public static void importData(CommandLine commandLine) { + long startTime = System.currentTimeMillis(); + createOptions(); + final HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setOptionComparator(null); + helpFormatter.setWidth(MAX_HELP_CONSOLE_WIDTH); + try { + parseBasicParams(commandLine); + parseSpecialParams(commandLine); + } catch (Exception e) { + IOT_PRINTER.println( + "Encounter an error when parsing the provided options: " + e.getMessage()); + System.exit(CODE_ERROR); + } + + IOT_PRINTER.println(isRemoteLoad ? "Load remotely." : "Load locally."); + + final int resultCode = importFromTargetPath(); + + ImportTsFileBase.printResult(startTime); + System.exit(resultCode); + } + + private static void parseSpecialParams(CommandLine commandLine) { + source = commandLine.getOptionValue(SOURCE_ARGS); + if (!Files.exists(Paths.get(source))) { + IOT_PRINTER.println(String.format("Source file or directory %s does not exist", source)); + System.exit(CODE_ERROR); + } + + final String onSuccess = commandLine.getOptionValue(ON_SUCCESS_ARGS).trim().toLowerCase(); + final String onFail = commandLine.getOptionValue(ON_FAIL_ARGS).trim().toLowerCase(); + if (!ImportTsFileOperation.isValidOperation(onSuccess) + || !ImportTsFileOperation.isValidOperation(onFail)) { + IOT_PRINTER.println("Args error: os/of must be one of none, mv, cp, delete"); + System.exit(CODE_ERROR); + } + + boolean isSuccessDirEqualsSourceDir = false; + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onSuccess) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onSuccess)) { + File dir = createSuccessDir(commandLine); + isSuccessDirEqualsSourceDir = isFileStoreEquals(source, dir); + } + + boolean isFailDirEqualsSourceDir = false; + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onFail) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onFail)) { + File dir = createFailDir(commandLine); + isFailDirEqualsSourceDir = isFileStoreEquals(source, dir); + } + + successOperation = ImportTsFileOperation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = ImportTsFileOperation.getOperation(onFail, isFailDirEqualsSourceDir); + + if (commandLine.getOptionValue(THREAD_NUM_ARGS) != null) { + threadNum = Integer.parseInt(commandLine.getOptionValue(THREAD_NUM_ARGS)); + } + + try { + isRemoteLoad = !NodeUrlUtils.containsLocalAddress(Collections.singletonList(host)); + } catch (UnknownHostException e) { + IOT_PRINTER.println( + "Unknown host: " + host + ". Exception: " + e.getMessage() + ". Will use remote load."); + } + if (commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS) != null) { + timestampPrecision = commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS); + } + + verify = + null != commandLine.getOptionValue(VERIFY_ARGS) + ? Boolean.parseBoolean(commandLine.getOptionValue(VERIFY_ARGS)) + : verify; + } + + public static boolean isFileStoreEquals(String pathString, File dir) { + try { + return Objects.equals( + Files.getFileStore(Paths.get(pathString)), Files.getFileStore(dir.toPath())); + } catch (IOException e) { + IOT_PRINTER.println("IOException when checking file store: " + e.getMessage()); + return false; + } + } + + public static File createSuccessDir(CommandLine commandLine) { + if (commandLine.getOptionValue(SUCCESS_DIR_ARGS) != null) { + successDir = commandLine.getOptionValue(SUCCESS_DIR_ARGS); + } + File file = new File(successDir); + if (!file.isDirectory()) { + if (!file.mkdirs()) { + IOT_PRINTER.println(String.format("Failed to create %s %s", SUCCESS_DIR_NAME, successDir)); + System.exit(CODE_ERROR); + } + } + return file; + } + + public static File createFailDir(CommandLine commandLine) { + if (commandLine.getOptionValue(FAIL_DIR_ARGS) != null) { + failDir = commandLine.getOptionValue(FAIL_DIR_ARGS); + } + File file = new File(failDir); + if (!file.isDirectory()) { + if (!file.mkdirs()) { + IOT_PRINTER.println(String.format("Failed to create %s %s", FAIL_DIR_NAME, failDir)); + System.exit(CODE_ERROR); + } + } + return file; + } + + public static int importFromTargetPath() { + try { + sessionPool = + new SessionPool.Builder() + .host(host) + .port(Integer.parseInt(port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .build(); + sessionPool.setEnableQueryRedirection(false); + checkTimePrecision(); + // set params + processSetParams(); + + ImportTsFileScanTool.traverseAndCollectFiles(); + ImportTsFileScanTool.addNoResourceOrModsToQueue(); + + asyncImportTsFiles(); + return CODE_OK; + } catch (InterruptedException e) { + IOT_PRINTER.println(String.format("Import tsfile fail: %s", e.getMessage())); + Thread.currentThread().interrupt(); + return CODE_ERROR; + } catch (Exception e) { + IOT_PRINTER.println(String.format("Import tsfile fail: %s", e.getMessage())); + return CODE_ERROR; + } finally { + if (sessionPool != null) { + sessionPool.close(); + } + } + } + + // process other classes need param + private static void processSetParams() { + // ImportTsFileLocally + final File file = new File(source); + ImportTsFileScanTool.setSourceFullPath(file.getAbsolutePath()); + if (!file.isFile() && !file.isDirectory()) { + IOT_PRINTER.println(String.format("Source file or directory %s does not exist", source)); + System.exit(CODE_ERROR); + } + + ImportTsFileLocally.setSessionPool(sessionPool); + ImportTsFileLocally.setVerify(verify); + + // ImportTsFileRemotely + ImportTsFileRemotely.setHost(host); + ImportTsFileRemotely.setPort(port); + ImportTsFileRemotely.setUsername(username); + ImportTsFileRemotely.setPassword(password); + ImportTsFileRemotely.setValidateTsFile(verify); + + // ImportTsFileBase + ImportTsFileBase.setSuccessAndFailDirAndOperation( + successDir, successOperation, failDir, failOperation); + } + + public static void asyncImportTsFiles() { + final List list = new ArrayList<>(threadNum); + for (int i = 0; i < threadNum; i++) { + final Thread thread = + new Thread( + isRemoteLoad + ? new ImportTsFileRemotely(timestampPrecision) + : new ImportTsFileLocally()); + thread.start(); + list.add(thread); + } + list.forEach( + thread -> { + try { + thread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + IOT_PRINTER.println("ImportTsFile thread join interrupted: " + e.getMessage()); + } + }); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java new file mode 100644 index 0000000000000..f1fd3bf2d30f5 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.tool.common.ImportTsFileOperation; + +import java.io.File; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Objects; +import java.util.concurrent.atomic.LongAdder; + +public abstract class ImportTsFileBase implements Runnable { + + private static final IoTPrinter IOT_PRINTER = new IoTPrinter(System.out); + + private static final LongAdder loadFileSuccessfulNum = new LongAdder(); + private static final LongAdder loadFileFailedNum = new LongAdder(); + private static final LongAdder processingLoadSuccessfulFileSuccessfulNum = new LongAdder(); + private static final LongAdder processingLoadFailedFileSuccessfulNum = new LongAdder(); + private static String timePrecision = "ms"; + private static String successDir; + private static ImportTsFileOperation successOperation; + private static String failDir; + private static ImportTsFileOperation failOperation; + + @Override + public void run() { + loadTsFile(); + } + + protected abstract void loadTsFile(); + + protected void processFailFile(final String filePath, final Exception e) { + // Reject because of memory controls, do retry later + try { + if (Objects.nonNull(e.getMessage()) && e.getMessage().contains("memory")) { + IOT_PRINTER.println( + "Rejecting file [ " + filePath + " ] due to memory constraints, will retry later."); + ImportTsFileScanTool.putToQueue(filePath); + return; + } + + loadFileFailedNum.increment(); + IOT_PRINTER.println("Failed to import [ " + filePath + " ] file: " + e.getMessage()); + + try { + processingFile(filePath, false); + processingLoadFailedFileSuccessfulNum.increment(); + IOT_PRINTER.println("Processed fail file [ " + filePath + " ] successfully!"); + } catch (final Exception processFailException) { + IOT_PRINTER.println( + "Failed to process fail file [ " + + filePath + + " ]: " + + processFailException.getMessage()); + } + } catch (final InterruptedException e1) { + IOT_PRINTER.println("Unexpected error occurred: " + e1.getMessage()); + Thread.currentThread().interrupt(); + } catch (final Exception e1) { + IOT_PRINTER.println("Unexpected error occurred: " + e1.getMessage()); + } + } + + protected static void processSuccessFile(final String filePath) { + loadFileSuccessfulNum.increment(); + IOT_PRINTER.println("Imported [ " + filePath + " ] file successfully!"); + + try { + processingFile(filePath, true); + processingLoadSuccessfulFileSuccessfulNum.increment(); + IOT_PRINTER.println("Processed success file [ " + filePath + " ] successfully!"); + } catch (final Exception processSuccessException) { + IOT_PRINTER.println( + "Failed to process success file [ " + + filePath + + " ]: " + + processSuccessException.getMessage()); + } + } + + public static void processingFile(final String filePath, final boolean isSuccess) { + final String relativePath = + filePath.substring(ImportTsFileScanTool.getSourceFullPathLength() + 1); + final Path sourcePath = Paths.get(filePath); + + final String target = + isSuccess + ? successDir + : failDir + File.separator + relativePath.replace(File.separator, "_"); + final Path targetPath = Paths.get(target); + + final String RESOURCE = ".resource"; + Path sourceResourcePath = Paths.get(sourcePath + RESOURCE); + sourceResourcePath = Files.exists(sourceResourcePath) ? sourceResourcePath : null; + final Path targetResourcePath = Paths.get(target + RESOURCE); + + final String MODS = ".mods"; + Path sourceModsPath = Paths.get(sourcePath + MODS); + sourceModsPath = Files.exists(sourceModsPath) ? sourceModsPath : null; + final Path targetModsPath = Paths.get(target + MODS); + + switch (isSuccess ? successOperation : failOperation) { + case DELETE: + { + try { + Files.deleteIfExists(sourcePath); + if (null != sourceResourcePath) { + Files.deleteIfExists(sourceResourcePath); + } + if (null != sourceModsPath) { + Files.deleteIfExists(sourceModsPath); + } + } catch (final Exception e) { + IOT_PRINTER.println(String.format("Failed to delete file: %s", e.getMessage())); + } + break; + } + case CP: + { + try { + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + if (null != sourceResourcePath) { + Files.copy( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + IOT_PRINTER.println(String.format("Failed to copy file: %s", e.getMessage())); + } + break; + } + case HARDLINK: + { + try { + Files.createLink(targetPath, sourcePath); + } catch (FileAlreadyExistsException e) { + IOT_PRINTER.println("Hardlink already exists: " + e.getMessage()); + } catch (final Exception e) { + try { + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (final Exception copyException) { + IOT_PRINTER.println( + String.format("Failed to copy file: %s", copyException.getMessage())); + } + } + + try { + if (null != sourceResourcePath) { + Files.copy( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + IOT_PRINTER.println( + String.format("Failed to copy resource or mods file: %s", e.getMessage())); + } + break; + } + case MV: + { + try { + Files.move(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + if (null != sourceResourcePath) { + Files.move( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.move(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + IOT_PRINTER.println(String.format("Failed to move file: %s", e.getMessage())); + } + break; + } + default: + break; + } + } + + protected static void printResult(final long startTime) { + IOT_PRINTER.println( + "Successfully load " + + loadFileSuccessfulNum.sum() + + " tsfile(s) (--on_success operation(s): " + + processingLoadSuccessfulFileSuccessfulNum.sum() + + " succeed, " + + (loadFileSuccessfulNum.sum() - processingLoadSuccessfulFileSuccessfulNum.sum()) + + " failed)"); + IOT_PRINTER.println( + "Failed to load " + + loadFileFailedNum.sum() + + " file(s) (--on_fail operation(s): " + + processingLoadFailedFileSuccessfulNum.sum() + + " succeed, " + + (loadFileFailedNum.sum() - processingLoadFailedFileSuccessfulNum.sum()) + + " failed)"); + IOT_PRINTER.println( + "Unprocessed " + + ImportTsFileScanTool.getTsFileQueueSize() + + " file(s), due to unexpected exceptions"); + IOT_PRINTER.println("For more details, please check the log."); + IOT_PRINTER.println( + "Total operation time: " + (System.currentTimeMillis() - startTime) + " ms."); + IOT_PRINTER.println("Work has been completed!"); + } + + public static void setSuccessAndFailDirAndOperation( + final String successDir, + final ImportTsFileOperation successOperation, + final String failDir, + final ImportTsFileOperation failOperation) { + ImportTsFileBase.successDir = successDir; + ImportTsFileBase.successOperation = successOperation; + ImportTsFileBase.failDir = failDir; + ImportTsFileBase.failOperation = failOperation; + } + + protected void setTimePrecision(String timePrecision) { + this.timePrecision = timePrecision; + } + + protected String getTimePrecision() { + return timePrecision; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileLocally.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileLocally.java new file mode 100644 index 0000000000000..f94050e98eb02 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileLocally.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.session.pool.SessionPool; + +public class ImportTsFileLocally extends ImportTsFileBase implements Runnable { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + private static SessionPool sessionPool; + private static boolean verify; + + @Override + public void loadTsFile() { + String filePath; + try { + while ((filePath = ImportTsFileScanTool.pollFromQueue()) != null) { + final String sql = + "load '" + filePath + "' onSuccess=none " + (verify ? "" : "verify=false"); + try { + sessionPool.executeNonQueryStatement(sql); + + processSuccessFile(filePath); + } catch (final Exception e) { + processFailFile(filePath, e); + } + } + } catch (final Exception e) { + ioTPrinter.println("Unexpected error occurred: " + e.getMessage()); + } + } + + public static void setSessionPool(SessionPool sessionPool) { + ImportTsFileLocally.sessionPool = sessionPool; + } + + public static void setVerify(boolean verify) { + ImportTsFileLocally.verify = verify; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java new file mode 100644 index 0000000000000..97b335afe2a49 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.connector.client.IoTDBSyncClient; +import org.apache.iotdb.commons.pipe.connector.payload.thrift.common.PipeTransferHandshakeConstant; +import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferFilePieceReq; +import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFilePieceReq; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFilePieceWithModReq; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFileSealReq; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFileSealWithModReq; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.pipe.api.exception.PipeConnectionException; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.thrift.transport.TTransportException; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +public class ImportTsFileRemotely extends ImportTsFileBase { + + private static final IoTPrinter IOT_PRINTER = new IoTPrinter(System.out); + + private static final String MODS = ".mods"; + private static final String LOAD_STRATEGY = "sync"; + private static final Integer MAX_RETRY_COUNT = 3; + + private IoTDBSyncClient client; + + private static String host; + private static String port; + + private static String username = SessionConfig.DEFAULT_USER; + private static String password = SessionConfig.DEFAULT_PASSWORD; + private static boolean validateTsFile; + + public ImportTsFileRemotely(String timePrecision) { + setTimePrecision(timePrecision); + initClient(); + sendHandshake(); + } + + @Override + public void loadTsFile() { + try { + String filePath; + while ((filePath = ImportTsFileScanTool.pollFromQueue()) != null) { + final File tsFile = new File(filePath); + try { + if (ImportTsFileScanTool.isContainModsFile(filePath + MODS)) { + doTransfer(tsFile, new File(filePath + MODS)); + } else { + doTransfer(tsFile, null); + } + + processSuccessFile(filePath); + } catch (final Exception e) { + IOT_PRINTER.println( + "Connect is abort, try to reconnect, max retry count: " + MAX_RETRY_COUNT); + + boolean isReconnectAndLoadSuccessFul = false; + + for (int i = 1; i <= MAX_RETRY_COUNT; i++) { + try { + IOT_PRINTER.println(String.format("The %sth retry will after %s seconds.", i, i * 2)); + LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(i * 2L)); + + close(); + initClient(); + sendHandshake(); + + if (ImportTsFileScanTool.isContainModsFile(filePath + MODS)) { + doTransfer(tsFile, new File(filePath + MODS)); + } else { + doTransfer(tsFile, null); + } + + processSuccessFile(filePath); + isReconnectAndLoadSuccessFul = true; + + IOT_PRINTER.println("Reconnect successful."); + break; + } catch (final Exception e1) { + IOT_PRINTER.println(String.format("The %sth reconnect failed", i)); + } + } + + if (!isReconnectAndLoadSuccessFul) { + processFailFile(filePath, e); + + close(); + initClient(); + sendHandshake(); + } + } + } + } catch (final Exception e) { + IOT_PRINTER.println("Unexpected error occurred: " + e.getMessage()); + } finally { + close(); + } + } + + public void sendHandshake() { + try { + final Map params = constructParamsMap(); + TPipeTransferResp resp = + client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params)); + + if (resp.getStatus().getCode() == TSStatusCode.PIPE_TYPE_ERROR.getStatusCode()) { + IOT_PRINTER.println( + String.format( + "Handshake error with target server ip: %s, port: %s, because: %s. " + + "Retry to handshake by PipeTransferHandshakeV1Req.", + client.getIpAddress(), client.getPort(), resp.getStatus())); + resp = + client.pipeTransfer( + PipeTransferDataNodeHandshakeV1Req.toTPipeTransferReq(getTimePrecision())); + } + + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new PipeConnectionException( + String.format( + "Handshake error with target server ip: %s, port: %s, because: %s.", + client.getIpAddress(), client.getPort(), resp.getStatus())); + } else { + client.setTimeout(PipeConfig.getInstance().getPipeConnectorTransferTimeoutMs()); + IOT_PRINTER.println( + String.format( + "Handshake success. Target server ip: %s, port: %s", + client.getIpAddress(), client.getPort())); + } + } catch (final Exception e) { + throw new PipeException( + String.format( + "Handshake error with target server ip: %s, port: %s, because: %s.", + client.getIpAddress(), client.getPort(), e.getMessage())); + } + } + + private Map constructParamsMap() { + final Map params = new HashMap<>(); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, getTimePrecision()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH, + Boolean.toString(true)); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, LOAD_STRATEGY); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, username); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, + Boolean.toString(validateTsFile)); + return params; + } + + public void doTransfer(final File tsFile, final File modFile) throws PipeException, IOException { + final TPipeTransferResp resp; + final TPipeTransferReq req; + + if (Objects.nonNull(modFile)) { + transferFilePieces(modFile, true); + transferFilePieces(tsFile, true); + + req = + PipeTransferTsFileSealWithModReq.toTPipeTransferReq( + modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length()); + } else { + transferFilePieces(tsFile, false); + + req = PipeTransferTsFileSealReq.toTPipeTransferReq(tsFile.getName(), tsFile.length()); + } + + try { + resp = client.pipeTransfer(req); + } catch (final Exception e) { + throw new PipeConnectionException( + String.format("Network error when seal file %s, because %s.", tsFile, e.getMessage()), e); + } + + final TSStatus status = resp.getStatus(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + throw new PipeConnectionException( + String.format("Seal file %s error, result status %s.", tsFile, status)); + } + + IOT_PRINTER.println("Successfully transferred file " + tsFile); + } + + private void transferFilePieces(final File file, final boolean isMultiFile) + throws PipeException, IOException { + final int readFileBufferSize = PipeConfig.getInstance().getPipeConnectorReadFileBufferSize(); + final byte[] readBuffer = new byte[readFileBufferSize]; + long position = 0; + try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) { + while (true) { + final int readLength = reader.read(readBuffer); + if (readLength == -1) { + break; + } + + final byte[] payLoad = + readLength == readFileBufferSize + ? readBuffer + : Arrays.copyOfRange(readBuffer, 0, readLength); + final PipeTransferFilePieceResp resp; + try { + final TPipeTransferReq req = + isMultiFile + ? getTransferMultiFilePieceReq(file.getName(), position, payLoad) + : getTransferSingleFilePieceReq(file.getName(), position, payLoad); + resp = PipeTransferFilePieceResp.fromTPipeTransferResp(client.pipeTransfer(req)); + } catch (final Exception e) { + throw new PipeConnectionException( + String.format( + "Network error when transfer file %s, because %s.", file, e.getMessage()), + e); + } + + position += readLength; + + final TSStatus status = resp.getStatus(); + if (status.getCode() == TSStatusCode.PIPE_TRANSFER_FILE_OFFSET_RESET.getStatusCode()) { + position = resp.getEndWritingOffset(); + reader.seek(position); + IOT_PRINTER.println(String.format("Redirect file position to %s.", position)); + continue; + } + + if (status.getCode() + == TSStatusCode.PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED.getStatusCode()) { + sendHandshake(); + } + // Only handle the failed statuses to avoid string format performance overhead + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + throw new PipeException( + String.format("Transfer file %s error, result status %s.", file, status)); + } + } + } + } + + private PipeTransferFilePieceReq getTransferMultiFilePieceReq( + final String fileName, final long position, final byte[] payLoad) throws IOException { + return PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, position, payLoad); + } + + private PipeTransferFilePieceReq getTransferSingleFilePieceReq( + final String fileName, final long position, final byte[] payLoad) throws IOException { + return PipeTransferTsFilePieceReq.toTPipeTransferReq(fileName, position, payLoad); + } + + private void initClient() { + try { + this.client = + new IoTDBSyncClient( + new ThriftClientProperty.Builder() + .setConnectionTimeoutMs( + PipeConfig.getInstance().getPipeConnectorHandshakeTimeoutMs()) + .setRpcThriftCompressionEnabled( + PipeConfig.getInstance().isPipeConnectorRPCThriftCompressionEnabled()) + .build(), + getEndPoint().getIp(), + getEndPoint().getPort(), + false, + "", + ""); + } catch (final TTransportException e) { + throw new PipeException("Sync client init error because " + e.getMessage()); + } + } + + private TEndPoint getEndPoint() { + return new TEndPoint(host, Integer.parseInt(port)); + } + + private String getClusterId() { + final SecureRandom random = new SecureRandom(); + final byte[] bytes = new byte[32]; // 32 bytes = 256 bits + random.nextBytes(bytes); + return "TSFILE-IMPORTER-" + UUID.nameUUIDFromBytes(bytes); + } + + private void close() { + try { + if (this.client != null) { + this.client.close(); + } + } catch (final Exception e) { + IOT_PRINTER.println("Failed to close client because " + e.getMessage()); + } + } + + public static void setHost(final String host) { + ImportTsFileRemotely.host = host; + } + + public static void setPort(final String port) { + ImportTsFileRemotely.port = port; + } + + public static void setUsername(final String username) { + ImportTsFileRemotely.username = username; + } + + public static void setPassword(final String password) { + ImportTsFileRemotely.password = password; + } + + public static void setValidateTsFile(final boolean validateTsFile) { + ImportTsFileRemotely.validateTsFile = validateTsFile; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java new file mode 100644 index 0000000000000..4891f42548616 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile; + +import java.io.File; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; + +public class ImportTsFileScanTool { + + private static final String RESOURCE = ".resource"; + private static final String MODS = ".mods"; + + private static final LinkedBlockingQueue tsfileQueue = new LinkedBlockingQueue<>(); + private static final Set tsfileSet = new HashSet<>(); + private static final Set resourceOrModsSet = new HashSet<>(); + private static String sourceFullPath; + + public static void traverseAndCollectFiles() throws InterruptedException { + traverseAndCollectFilesBySourceFullPath(new File(sourceFullPath)); + } + + private static void traverseAndCollectFilesBySourceFullPath(final File file) + throws InterruptedException { + if (file.isFile()) { + if (file.getName().endsWith(RESOURCE) || file.getName().endsWith(MODS)) { + resourceOrModsSet.add(file.getAbsolutePath()); + } else { + tsfileSet.add(file.getAbsolutePath()); + tsfileQueue.put(file.getAbsolutePath()); + } + } else if (file.isDirectory()) { + final File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + traverseAndCollectFilesBySourceFullPath(f); + } + } + } + } + + public static void addNoResourceOrModsToQueue() throws InterruptedException { + for (final String filePath : resourceOrModsSet) { + final String tsfilePath = + filePath.endsWith(RESOURCE) + ? filePath.substring(0, filePath.length() - RESOURCE.length()) + : filePath.substring(0, filePath.length() - MODS.length()); + if (!tsfileSet.contains(tsfilePath)) { + tsfileQueue.put(filePath); + } + } + } + + public static boolean isContainModsFile(final String modsFilePath) { + return ImportTsFileScanTool.resourceOrModsSet.contains(modsFilePath); + } + + public static String pollFromQueue() { + return ImportTsFileScanTool.tsfileQueue.poll(); + } + + public static void putToQueue(final String filePath) throws InterruptedException { + ImportTsFileScanTool.tsfileQueue.put(filePath); + } + + public static void setSourceFullPath(final String sourceFullPath) { + ImportTsFileScanTool.sourceFullPath = sourceFullPath; + } + + public static int getSourceFullPathLength() { + return new File(sourceFullPath).isDirectory() + ? ImportTsFileScanTool.sourceFullPath.length() + : new File(ImportTsFileScanTool.sourceFullPath).getParent().length(); + } + + public static int getTsFileQueueSize() { + return ImportTsFileScanTool.tsfileQueue.size(); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/AbstractSubscriptionTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/AbstractSubscriptionTsFile.java new file mode 100644 index 0000000000000..aa962044cc27d --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/AbstractSubscriptionTsFile.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile.subscription; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder; +import org.apache.iotdb.tool.common.Constants; + +import java.util.concurrent.ExecutorService; + +public abstract class AbstractSubscriptionTsFile { + + protected static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + protected static CommonParam commonParam = CommonParam.getInstance(); + + public static void setSubscriptionSession() throws IoTDBConnectionException { + if (Constants.TABLE_MODEL.equalsIgnoreCase(commonParam.getSqlDialect())) { + commonParam.setSubscriptionTsFile(new SubscriptionTableTsFile()); + commonParam.setTableSubs( + new SubscriptionTableSessionBuilder() + .host(commonParam.getSrcHost()) + .port(commonParam.getSrcPort()) + .username(commonParam.getSrcUserName()) + .password(commonParam.getSrcPassword()) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .build()); + commonParam.getTableSubs().open(); + } else { + + commonParam.setSubscriptionTsFile(new SubscriptionTreeTsFile()); + commonParam.setTreeSubs( + new SubscriptionTreeSessionBuilder() + .host(commonParam.getSrcHost()) + .port(commonParam.getSrcPort()) + .username(commonParam.getSrcUserName()) + .password(commonParam.getSrcPassword()) + .thriftMaxFrameSize(SessionConfig.DEFAULT_MAX_FRAME_SIZE) + .build()); + commonParam.getTreeSubs().open(); + } + } + + public abstract void createTopics(String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + public abstract void doClean() throws Exception; + + public abstract void createConsumers(String groupId); + + public abstract void subscribe(String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + public abstract void consumerPoll(ExecutorService executor, String topicName); +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/CommonParam.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/CommonParam.java new file mode 100644 index 0000000000000..34b26a6db02d8 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/CommonParam.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile.subscription; + +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.ISubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class CommonParam { + + private static AtomicInteger countFile = new AtomicInteger(0); + private static String path = "root.**"; + private static String table = ".*"; + private static String database = ".*"; + private static int startIndex = 0; + private static int consumerCount = 8; + private static ISubscriptionTableSession tableSubs; + private static ISubscriptionTreeSession treeSubs; + + private static String pathFull = "root.**"; + private static String srcHost = "127.0.0.1"; + private static int srcPort = 6667; + private static String srcUserName = "root"; + private static String srcPassword = "root"; + private static String sqlDialect = "tree"; + private static String startTime = ""; + private static String endTime = ""; + private static String targetDir = "target"; + private static List pullTableConsumers; + private static List pullTreeConsumers; + + private static AbstractSubscriptionTsFile subscriptionTsFile; + + private static CommonParam instance; + + public static synchronized CommonParam getInstance() { + if (null == instance) { + instance = new CommonParam(); + } + return instance; + } + + public static String getPath() { + return path; + } + + public static void setPath(String path) { + CommonParam.path = path; + } + + public static String getTable() { + return table; + } + + public static void setTable(String table) { + CommonParam.table = table; + } + + public static String getDatabase() { + return database; + } + + public static void setDatabase(String database) { + CommonParam.database = database; + } + + public static int getStartIndex() { + return startIndex; + } + + public static int getConsumerCount() { + return consumerCount; + } + + public static void setConsumerCount(int consumerCount) { + CommonParam.consumerCount = consumerCount; + } + + public static ISubscriptionTableSession getTableSubs() { + return tableSubs; + } + + public static void setTableSubs(ISubscriptionTableSession tableSubs) { + CommonParam.tableSubs = tableSubs; + } + + public static ISubscriptionTreeSession getTreeSubs() { + return treeSubs; + } + + public static void setTreeSubs(ISubscriptionTreeSession treeSubs) { + CommonParam.treeSubs = treeSubs; + } + + public static String getPathFull() { + return pathFull; + } + + public static String getSrcHost() { + return srcHost; + } + + public static void setSrcHost(String srcHost) { + CommonParam.srcHost = srcHost; + } + + public static int getSrcPort() { + return srcPort; + } + + public static void setSrcPort(int srcPort) { + CommonParam.srcPort = srcPort; + } + + public static String getSrcUserName() { + return srcUserName; + } + + public static void setSrcUserName(String srcUserName) { + CommonParam.srcUserName = srcUserName; + } + + public static String getSrcPassword() { + return srcPassword; + } + + public static void setSrcPassword(String srcPassword) { + CommonParam.srcPassword = srcPassword; + } + + public static String getSqlDialect() { + return sqlDialect; + } + + public static void setSqlDialect(String sqlDialect) { + CommonParam.sqlDialect = sqlDialect; + } + + public static String getStartTime() { + return startTime; + } + + public static void setStartTime(String startTime) { + CommonParam.startTime = startTime; + } + + public static String getEndTime() { + return endTime; + } + + public static void setEndTime(String endTime) { + CommonParam.endTime = endTime; + } + + public static String getTargetDir() { + return targetDir; + } + + public static void setTargetDir(String targetDir) { + CommonParam.targetDir = targetDir; + } + + public static List getPullTableConsumers() { + return pullTableConsumers; + } + + public static void setPullTableConsumers( + List pullTableConsumers) { + CommonParam.pullTableConsumers = pullTableConsumers; + } + + public static List getPullTreeConsumers() { + return pullTreeConsumers; + } + + public static void setPullTreeConsumers(List pullTreeConsumers) { + CommonParam.pullTreeConsumers = pullTreeConsumers; + } + + public static AbstractSubscriptionTsFile getSubscriptionTsFile() { + return subscriptionTsFile; + } + + public static void setSubscriptionTsFile(AbstractSubscriptionTsFile subscriptionTsFile) { + CommonParam.subscriptionTsFile = subscriptionTsFile; + } + + public static AtomicInteger getCountFile() { + return countFile; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/SubscriptionTableTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/SubscriptionTableTsFile.java new file mode 100644 index 0000000000000..f869490e7c56f --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/SubscriptionTableTsFile.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile.subscription; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder; +import org.apache.iotdb.session.subscription.model.Topic; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutorService; + +import static java.lang.System.out; + +public class SubscriptionTableTsFile extends AbstractSubscriptionTsFile { + + @Override + public void createTopics(String topicName) + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.MODE_KEY, Constants.MODE); + properties.put(TopicConstant.FORMAT_KEY, Constants.HANDLER); + properties.put(TopicConstant.STRICT_KEY, Constants.STRICT); + properties.put(TopicConstant.LOOSE_RANGE_KEY, Constants.LOOSE_RANGE); + if (StringUtils.isNotBlank(commonParam.getStartTime())) + properties.put(TopicConstant.START_TIME_KEY, commonParam.getStartTime()); + if (StringUtils.isNotBlank(commonParam.getEndTime())) + properties.put(TopicConstant.END_TIME_KEY, commonParam.getEndTime()); + if (StringUtils.isNotBlank(commonParam.getDatabase()) + || StringUtils.isNotBlank(commonParam.getTable())) { + properties.put(TopicConstant.DATABASE_KEY, commonParam.getDatabase()); + properties.put(TopicConstant.TABLE_KEY, commonParam.getTable()); + } + commonParam.getTableSubs().createTopic(topicName, properties); + } + + @Override + public void doClean() throws Exception { + List pullTableConsumers = commonParam.getPullTableConsumers(); + for (int i = commonParam.getStartIndex(); i < pullTableConsumers.size(); i++) { + SubscriptionTablePullConsumer consumer = + (SubscriptionTablePullConsumer) pullTableConsumers.get(i); + String path = + commonParam.getTargetDir() + + File.separator + + consumer.getConsumerGroupId() + + File.separator + + consumer.getConsumerId(); + File file = new File(path); + if (file.exists()) { + FileUtils.deleteFileOrDirectory(file); + } + } + for (Topic topic : CommonParam.getTableSubs().getTopics()) { + try { + commonParam.getTableSubs().dropTopicIfExists(topic.getTopicName()); + } catch (Exception e) { + + } + } + commonParam.getTableSubs().close(); + } + + @Override + public void createConsumers(String groupId) { + commonParam.setPullTableConsumers(new ArrayList<>(CommonParam.getConsumerCount())); + for (int i = commonParam.getStartIndex(); i < commonParam.getConsumerCount(); i++) { + commonParam + .getPullTableConsumers() + .add( + new SubscriptionTablePullConsumerBuilder() + .host(commonParam.getSrcHost()) + .port(commonParam.getSrcPort()) + .consumerId(Constants.CONSUMER_NAME_PREFIX + i) + .consumerGroupId(groupId) + .autoCommit(Constants.AUTO_COMMIT) + .autoCommitIntervalMs(Constants.AUTO_COMMIT_INTERVAL) + .fileSaveDir(commonParam.getTargetDir()) + .build()); + } + commonParam + .getPullTableConsumers() + .removeIf( + consumer -> { + try { + consumer.open(); + return false; + } catch (SubscriptionException e) { + return true; + } + }); + commonParam.setConsumerCount(commonParam.getPullTableConsumers().size()); + } + + @Override + public void subscribe(String topicName) + throws IoTDBConnectionException, StatementExecutionException { + List pullTableConsumers = commonParam.getPullTableConsumers(); + for (int i = 0; i < pullTableConsumers.size(); i++) { + try { + pullTableConsumers.get(i).subscribe(topicName); + } catch (Exception e) { + e.printStackTrace(out); + } + } + } + + @Override + public void consumerPoll(ExecutorService executor, String topicName) { + List pullTableConsumers = commonParam.getPullTableConsumers(); + for (int i = commonParam.getStartIndex(); i < pullTableConsumers.size(); i++) { + SubscriptionTablePullConsumer consumer = + (SubscriptionTablePullConsumer) pullTableConsumers.get(i); + executor.submit( + new Runnable() { + @Override + public void run() { + final String consumerGroupId = consumer.getConsumerGroupId(); + while (!consumer.allTopicMessagesHaveBeenConsumed()) { + try { + for (final SubscriptionMessage message : + consumer.poll(Constants.POLL_MESSAGE_TIMEOUT)) { + final SubscriptionTsFileHandler handler = message.getTsFileHandler(); + ioTPrinter.println(handler.getFile().getName()); + try { + handler.moveFile( + Paths.get( + commonParam.getTargetDir() + File.separator + consumerGroupId, + handler.getPath().getFileName().toString())); + } catch (IOException e) { + throw new RuntimeException(e); + } + commonParam.getCountFile().incrementAndGet(); + } + } catch (Exception e) { + e.printStackTrace(System.out); + } + } + consumer.unsubscribe(topicName); + } + }); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/SubscriptionTreeTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/SubscriptionTreeTsFile.java new file mode 100644 index 0000000000000..7f2d302e6e413 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/subscription/SubscriptionTreeTsFile.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.tsfile.subscription; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.model.Topic; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutorService; + +import static java.lang.System.out; + +public class SubscriptionTreeTsFile extends AbstractSubscriptionTsFile { + + @Override + public void createTopics(String topicName) + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + properties.put(TopicConstant.MODE_KEY, Constants.MODE); + properties.put(TopicConstant.FORMAT_KEY, Constants.HANDLER); + properties.put(TopicConstant.STRICT_KEY, Constants.STRICT); + properties.put(TopicConstant.LOOSE_RANGE_KEY, Constants.LOOSE_RANGE); + if (StringUtils.isNotBlank(commonParam.getStartTime())) + properties.put(TopicConstant.START_TIME_KEY, commonParam.getStartTime()); + if (StringUtils.isNotBlank(commonParam.getEndTime())) + properties.put(TopicConstant.END_TIME_KEY, commonParam.getEndTime()); + properties.put( + TopicConstant.PATH_KEY, + StringUtils.isNotBlank(commonParam.getPath()) + ? commonParam.getPath() + : commonParam.getPathFull()); + commonParam.getTreeSubs().createTopic(topicName, properties); + } + + @Override + public void doClean() throws Exception { + List pullTreeConsumers = commonParam.getPullTreeConsumers(); + for (int i = commonParam.getStartIndex(); i < pullTreeConsumers.size(); i++) { + SubscriptionTreePullConsumer consumer = pullTreeConsumers.get(i); + String path = + commonParam.getTargetDir() + + File.separator + + consumer.getConsumerGroupId() + + File.separator + + consumer.getConsumerId(); + File file = new File(path); + if (file.exists()) { + FileUtils.deleteFileOrDirectory(file); + } + } + for (Topic topic : commonParam.getTreeSubs().getTopics()) { + try { + commonParam.getTreeSubs().dropTopicIfExists(topic.getTopicName()); + } catch (Exception e) { + + } + } + commonParam.getTreeSubs().close(); + } + + @Override + public void createConsumers(String groupId) { + commonParam.setPullTreeConsumers(new ArrayList<>(commonParam.getConsumerCount())); + for (int i = commonParam.getStartIndex(); i < commonParam.getConsumerCount(); i++) { + commonParam + .getPullTreeConsumers() + .add( + new SubscriptionTreePullConsumer.Builder() + .host(commonParam.getSrcHost()) + .port(commonParam.getSrcPort()) + .consumerId(Constants.CONSUMER_NAME_PREFIX + i) + .consumerGroupId(groupId) + .autoCommit(Constants.AUTO_COMMIT) + .autoCommitIntervalMs(Constants.AUTO_COMMIT_INTERVAL) + .fileSaveDir(commonParam.getTargetDir()) + .buildPullConsumer()); + } + commonParam + .getPullTreeConsumers() + .removeIf( + consumer -> { + try { + consumer.open(); + return false; + } catch (SubscriptionException e) { + return true; + } + }); + commonParam.setConsumerCount(commonParam.getPullTreeConsumers().size()); + } + + @Override + public void subscribe(String topicName) + throws IoTDBConnectionException, StatementExecutionException { + List pullTreeConsumers = commonParam.getPullTreeConsumers(); + for (int i = 0; i < pullTreeConsumers.size(); i++) { + try { + pullTreeConsumers.get(i).subscribe(topicName); + } catch (Exception e) { + e.printStackTrace(out); + } + } + } + + @Override + public void consumerPoll(ExecutorService executor, String topicName) { + List pullTreeConsumers = commonParam.getPullTreeConsumers(); + for (int i = commonParam.getStartIndex(); i < pullTreeConsumers.size(); i++) { + SubscriptionTreePullConsumer consumer = commonParam.getPullTreeConsumers().get(i); + final String consumerGroupId = consumer.getConsumerGroupId(); + executor.submit( + new Runnable() { + @Override + public void run() { + while (!consumer.allTopicMessagesHaveBeenConsumed()) { + try { + for (final SubscriptionMessage message : + consumer.poll(Constants.POLL_MESSAGE_TIMEOUT)) { + final SubscriptionTsFileHandler handler = message.getTsFileHandler(); + ioTPrinter.println(handler.getFile().getName()); + try { + handler.moveFile( + Paths.get( + commonParam.getTargetDir() + File.separator + consumerGroupId, + handler.getPath().getFileName().toString())); + } catch (IOException e) { + throw new RuntimeException(e); + } + commonParam.getCountFile().incrementAndGet(); + } + } catch (Exception e) { + e.printStackTrace(System.out); + } + } + consumer.unsubscribe(topicName); + } + }); + } + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java new file mode 100644 index 0000000000000..63541f586362b --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool; + +import org.apache.iotdb.tool.common.ImportTsFileOperation; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ImportTsFileOperationTest { + + @Test + public void testIsValidOperation() { + assertTrue(ImportTsFileOperation.isValidOperation("none")); + assertTrue(ImportTsFileOperation.isValidOperation("mv")); + assertTrue(ImportTsFileOperation.isValidOperation("cp")); + assertTrue(ImportTsFileOperation.isValidOperation("delete")); + assertFalse(ImportTsFileOperation.isValidOperation("invalid")); + } + + @Test + public void testGetOperation() { + assertEquals(ImportTsFileOperation.NONE, ImportTsFileOperation.getOperation("none", false)); + assertEquals(ImportTsFileOperation.MV, ImportTsFileOperation.getOperation("mv", false)); + assertEquals(ImportTsFileOperation.HARDLINK, ImportTsFileOperation.getOperation("cp", true)); + assertEquals(ImportTsFileOperation.CP, ImportTsFileOperation.getOperation("cp", false)); + assertEquals(ImportTsFileOperation.DELETE, ImportTsFileOperation.getOperation("delete", false)); + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteDataFileTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteDataFileTest.java index eb9e01d293700..f9b43b8a88079 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteDataFileTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteDataFileTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.tool; +import org.apache.iotdb.tool.data.AbstractDataTool; + import org.junit.Before; import org.junit.Test; diff --git a/iotdb-client/client-cpp/README.md b/iotdb-client/client-cpp/README.md index 81983e06e2e4d..db372bbae110f 100644 --- a/iotdb-client/client-cpp/README.md +++ b/iotdb-client/client-cpp/README.md @@ -42,12 +42,21 @@ Studio. It is highly recommended to use Visual Studio 2022 or later. If you are using Visual Studio 2022, you can compile the cpp client with the following command: ``` -mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTest +mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests -D"boost.include.dir"="D:\boost_1_75_0" -D"boost.library.dir"="D:\boost_1_75_0\stage\lib" ``` -#### Visual Studio 2019 or older -If you are using Visual Studio 2019 or older, the pre-built Thrift library is incompatible. You +##### Visual Studio 2019 +If you are using Visual Studio 2019, you can compile the cpp client with the following command: + +``` +mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests +-D"boost.include.dir"="D:\boost_1_75_0" -D"boost.library.dir"="D:\boost_1_75_0\stage\lib" +-Diotdb-tools-thrift.version=0.14.1.1-msvc142-SNAPSHOT -Dcmake.generator="Visual Studio 16 2019" +``` + +#### Visual Studio 2017 or older +If you are using Visual Studio 2017 or older, the pre-built Thrift library is incompatible. You will have to compile the thrift library manually: 1. Install the dependencies of Thrift: @@ -70,7 +79,7 @@ will have to compile the thrift library manually: 5. Return to the cpp client repository and compile it with: ``` -mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTest +mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests -D"boost.include.dir"="D:\boost_1_75_0" -D"boost.library.dir"="D:\boost_1_75_0\stage\lib" ``` diff --git a/iotdb-client/client-cpp/pom.xml b/iotdb-client/client-cpp/pom.xml index 6aa597170f3a5..62a350e88574c 100644 --- a/iotdb-client/client-cpp/pom.xml +++ b/iotdb-client/client-cpp/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT client-cpp pom @@ -43,7 +43,7 @@ org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT provided @@ -163,6 +163,58 @@ ${project.basedir}/src/main/Session.cpp ${project.build.directory}/build/main/generated-sources-cpp/Session.cpp + + ${project.basedir}/src/main/TableSession.h + ${project.build.directory}/build/main/generated-sources-cpp/TableSession.h + + + ${project.basedir}/src/main/TableSession.cpp + ${project.build.directory}/build/main/generated-sources-cpp/TableSession.cpp + + + ${project.basedir}/src/main/TableSessionBuilder.h + ${project.build.directory}/build/main/generated-sources-cpp/TableSessionBuilder.h + + + ${project.basedir}/src/main/NodesSupplier.h + ${project.build.directory}/build/main/generated-sources-cpp/NodesSupplier.h + + + ${project.basedir}/src/main/NodesSupplier.cpp + ${project.build.directory}/build/main/generated-sources-cpp/NodesSupplier.cpp + + + ${project.basedir}/src/main/ThriftConnection.h + ${project.build.directory}/build/main/generated-sources-cpp/ThriftConnection.h + + + ${project.basedir}/src/main/ThriftConnection.cpp + ${project.build.directory}/build/main/generated-sources-cpp/ThriftConnection.cpp + + + ${project.basedir}/src/main/SessionConnection.h + ${project.build.directory}/build/main/generated-sources-cpp/SessionConnection.h + + + ${project.basedir}/src/main/SessionConnection.cpp + ${project.build.directory}/build/main/generated-sources-cpp/SessionConnection.cpp + + + ${project.basedir}/src/main/AbstractSessionBuilder.h + ${project.build.directory}/build/main/generated-sources-cpp/AbstractSessionBuilder.h + + + ${project.basedir}/src/main/Common.h + ${project.build.directory}/build/main/generated-sources-cpp/Common.h + + + ${project.basedir}/src/main/Common.cc + ${project.build.directory}/build/main/generated-sources-cpp/Common.cc + + + ${project.basedir}/src/main/DeviceID.h + ${project.build.directory}/build/main/generated-sources-cpp/DeviceID.h + diff --git a/iotdb-client/client-cpp/src/main/AbstractSessionBuilder.h b/iotdb-client/client-cpp/src/main/AbstractSessionBuilder.h new file mode 100644 index 0000000000000..441e3a41f4fac --- /dev/null +++ b/iotdb-client/client-cpp/src/main/AbstractSessionBuilder.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_ABSTRACTSESSIONBUILDER_H +#define IOTDB_ABSTRACTSESSIONBUILDER_H + +#include + +class AbstractSessionBuilder { +public: + std::string host = "localhost"; + int rpcPort = 6667; + std::string username = "root"; + std::string password = "root"; + std::string zoneId = ""; + int fetchSize = 10000; + std::string sqlDialect = "tree"; + std::string database = ""; + bool enableAutoFetch = true; + bool enableRedirections = true; + bool enableRPCCompression = false; +}; + +#endif // IOTDB_ABSTRACTSESSIONBUILDER_H \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/Common.cc b/iotdb-client/client-cpp/src/main/Common.cc new file mode 100644 index 0000000000000..2ec900f7d7e65 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/Common.cc @@ -0,0 +1,186 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Common.h" +#include + +int32_t parseDateExpressionToInt(const boost::gregorian::date& date) { + if(date.is_not_a_date()) { + throw IoTDBException("Date expression is null or empty."); + } + + const int year = date.year(); + if(year < 1000 || year > 9999) { + throw DateTimeParseException( + "Year must be between 1000 and 9999.", + boost::gregorian::to_iso_extended_string(date), + 0 + ); + } + + const int64_t result = static_cast(year) * 10000 + + date.month() * 100 + + date.day(); + if(result > INT32_MAX || result < INT32_MIN) { + throw DateTimeParseException( + "Date value overflow. ", + boost::gregorian::to_iso_extended_string(date), + 0 + ); + } + return static_cast(result); +} + +boost::gregorian::date parseIntToDate(int32_t dateInt) { + if (dateInt == EMPTY_DATE_INT) { + return boost::gregorian::date(boost::date_time::not_a_date_time); + } + int year = dateInt / 10000; + int month = (dateInt % 10000) / 100; + int day = dateInt % 100; + return boost::gregorian::date(year, month, day); +} + +void RpcUtils::verifySuccess(const TSStatus &status) { + if (status.code == TSStatusCode::MULTIPLE_ERROR) { + verifySuccess(status.subStatus); + return; + } + if (status.code != TSStatusCode::SUCCESS_STATUS + && status.code != TSStatusCode::REDIRECTION_RECOMMEND) { + throw ExecutionException(to_string(status.code) + ": " + status.message, status); + } +} + +void RpcUtils::verifySuccessWithRedirection(const TSStatus &status) { + verifySuccess(status); + if (status.__isset.redirectNode) { + throw RedirectException(to_string(status.code) + ": " + status.message, status.redirectNode); + } + if (status.__isset.subStatus) { + auto statusSubStatus = status.subStatus; + vector endPointList(statusSubStatus.size()); + int count = 0; + for (TSStatus subStatus : statusSubStatus) { + if (subStatus.__isset.redirectNode) { + endPointList[count++] = subStatus.redirectNode; + } else { + TEndPoint endPoint; + endPointList[count++] = endPoint; + } + } + if (!endPointList.empty()) { + throw RedirectException(to_string(status.code) + ": " + status.message, endPointList); + } + } +} + +void RpcUtils::verifySuccessWithRedirectionForMultiDevices(const TSStatus &status, vector devices) { + verifySuccess(status); + + if (status.code == TSStatusCode::MULTIPLE_ERROR + || status.code == TSStatusCode::REDIRECTION_RECOMMEND) { + map deviceEndPointMap; + vector statusSubStatus; + for (int i = 0; i < statusSubStatus.size(); i++) { + TSStatus subStatus = statusSubStatus[i]; + if (subStatus.__isset.redirectNode) { + deviceEndPointMap.insert(make_pair(devices[i], subStatus.redirectNode)); + } + } + throw RedirectException(to_string(status.code) + ": " + status.message, deviceEndPointMap); + } + + if (status.__isset.redirectNode) { + throw RedirectException(to_string(status.code) + ": " + status.message, status.redirectNode); + } + if (status.__isset.subStatus) { + auto statusSubStatus = status.subStatus; + vector endPointList(statusSubStatus.size()); + int count = 0; + for (TSStatus subStatus : statusSubStatus) { + if (subStatus.__isset.redirectNode) { + endPointList[count++] = subStatus.redirectNode; + } else { + TEndPoint endPoint; + endPointList[count++] = endPoint; + } + } + if (!endPointList.empty()) { + throw RedirectException(to_string(status.code) + ": " + status.message, endPointList); + } + } +} + +void RpcUtils::verifySuccess(const vector &statuses) { + for (const TSStatus &status: statuses) { + if (status.code != TSStatusCode::SUCCESS_STATUS) { + throw BatchExecutionException(status.message, statuses); + } + } +} + +TSStatus RpcUtils::getStatus(TSStatusCode::TSStatusCode tsStatusCode) { + TSStatus status; + status.__set_code(tsStatusCode); + return status; +} + +TSStatus RpcUtils::getStatus(int code, const string &message) { + TSStatus status; + status.__set_code(code); + status.__set_message(message); + return status; +} + +shared_ptr RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode) { + TSStatus status = getStatus(tsStatusCode); + return getTSExecuteStatementResp(status); +} + +shared_ptr +RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, const string &message) { + TSStatus status = getStatus(tsStatusCode, message); + return getTSExecuteStatementResp(status); +} + +shared_ptr RpcUtils::getTSExecuteStatementResp(const TSStatus &status) { + shared_ptr resp(new TSExecuteStatementResp()); + TSStatus tsStatus(status); + resp->__set_status(status); + return resp; +} + +shared_ptr RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode) { + TSStatus status = getStatus(tsStatusCode); + return getTSFetchResultsResp(status); +} + +shared_ptr +RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, const string &appendMessage) { + TSStatus status = getStatus(tsStatusCode, appendMessage); + return getTSFetchResultsResp(status); +} + +shared_ptr RpcUtils::getTSFetchResultsResp(const TSStatus &status) { + shared_ptr resp(new TSFetchResultsResp()); + TSStatus tsStatus(status); + resp->__set_status(tsStatus); + return resp; +} \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/Common.h b/iotdb-client/client-cpp/src/main/Common.h new file mode 100644 index 0000000000000..dbabb1b5657dd --- /dev/null +++ b/iotdb-client/client-cpp/src/main/Common.h @@ -0,0 +1,390 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_COMMON_H +#define IOTDB_COMMON_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "client_types.h" +#include "common_types.h" + +using namespace std; + +using ::apache::thrift::protocol::TBinaryProtocol; +using ::apache::thrift::protocol::TCompactProtocol; +using ::apache::thrift::transport::TSocket; +using ::apache::thrift::transport::TTransport; +using ::apache::thrift::transport::TTransportException; +using ::apache::thrift::transport::TBufferedTransport; +using ::apache::thrift::transport::TFramedTransport; +using ::apache::thrift::TException; + +using namespace std; + +constexpr int32_t EMPTY_DATE_INT = 10000101; + +int32_t parseDateExpressionToInt(const boost::gregorian::date& date); +boost::gregorian::date parseIntToDate(int32_t dateInt); + +namespace Version { +enum Version { + V_0_12, V_0_13, V_1_0 +}; +} + +namespace CompressionType { +enum CompressionType { + UNCOMPRESSED = (char)0, + SNAPPY = (char)1, + GZIP = (char)2, + LZO = (char)3, + SDT = (char)4, + PAA = (char)5, + PLA = (char)6, + LZ4 = (char)7, + ZSTD = (char)8, + LZMA2 = (char)9, +}; +} + +namespace TSDataType { +enum TSDataType { + BOOLEAN = (char)0, + INT32 = (char)1, + INT64 = (char)2, + FLOAT = (char)3, + DOUBLE = (char)4, + TEXT = (char)5, + VECTOR = (char)6, + UNKNOWN = (char)7, + TIMESTAMP = (char)8, + DATE = (char)9, + BLOB = (char)10, + STRING = (char)11, + NULLTYPE = (char)254, + INVALID_DATATYPE = (char)255 +}; +} + +namespace TSEncoding { +enum TSEncoding { + PLAIN = (char)0, + DICTIONARY = (char)1, + RLE = (char)2, + DIFF = (char)3, + TS_2DIFF = (char)4, + BITMAP = (char)5, + GORILLA_V1 = (char)6, + REGULAR = (char)7, + GORILLA = (char)8, + ZIGZAG = (char)9, + FREQ = (char)10, + INVALID_ENCODING = (char)255 +}; +} + +namespace TSStatusCode { +enum TSStatusCode { + SUCCESS_STATUS = 200, + + // System level + INCOMPATIBLE_VERSION = 201, + CONFIGURATION_ERROR = 202, + START_UP_ERROR = 203, + SHUT_DOWN_ERROR = 204, + + // General Error + UNSUPPORTED_OPERATION = 300, + EXECUTE_STATEMENT_ERROR = 301, + MULTIPLE_ERROR = 302, + ILLEGAL_PARAMETER = 303, + OVERLAP_WITH_EXISTING_TASK = 304, + INTERNAL_SERVER_ERROR = 305, + + // Client, + REDIRECTION_RECOMMEND = 400, + + // Schema Engine + DATABASE_NOT_EXIST = 500, + DATABASE_ALREADY_EXISTS = 501, + SERIES_OVERFLOW = 502, + TIMESERIES_ALREADY_EXIST = 503, + TIMESERIES_IN_BLACK_LIST = 504, + ALIAS_ALREADY_EXIST = 505, + PATH_ALREADY_EXIST = 506, + METADATA_ERROR = 507, + PATH_NOT_EXIST = 508, + ILLEGAL_PATH = 509, + CREATE_TEMPLATE_ERROR = 510, + DUPLICATED_TEMPLATE = 511, + UNDEFINED_TEMPLATE = 512, + TEMPLATE_NOT_SET = 513, + DIFFERENT_TEMPLATE = 514, + TEMPLATE_IS_IN_USE = 515, + TEMPLATE_INCOMPATIBLE = 516, + SEGMENT_NOT_FOUND = 517, + PAGE_OUT_OF_SPACE = 518, + RECORD_DUPLICATED = 519, + SEGMENT_OUT_OF_SPACE = 520, + PBTREE_FILE_NOT_EXISTS = 521, + OVERSIZE_RECORD = 522, + PBTREE_FILE_REDO_LOG_BROKEN = 523, + TEMPLATE_NOT_ACTIVATED = 524, + + // Storage Engine + SYSTEM_READ_ONLY = 600, + STORAGE_ENGINE_ERROR = 601, + STORAGE_ENGINE_NOT_READY = 602, + + // Query Engine + PLAN_FAILED_NETWORK_PARTITION = 721 +}; +} + +class IoTDBException : public std::exception { +public: + IoTDBException() = default; + + explicit IoTDBException(const std::string& m) : message(m) { + } + + explicit IoTDBException(const char* m) : message(m) { + } + + virtual const char* what() const noexcept override { + return message.c_str(); + } + +private: + std::string message; +}; + +class DateTimeParseException : public IoTDBException { +private: + std::string parsedString; + int errorIndex; + +public: + explicit DateTimeParseException(const std::string& message, + std::string parsedData, + int errorIndex) + : IoTDBException(message), + parsedString(std::move(parsedData)), + errorIndex(errorIndex) {} + + explicit DateTimeParseException(const std::string& message, + std::string parsedData, + int errorIndex, + const std::exception& cause) + : IoTDBException(message + " [Caused by: " + cause.what() + "]"), + parsedString(std::move(parsedData)), + errorIndex(errorIndex) {} + + const std::string& getParsedString() const noexcept { + return parsedString; + } + + int getErrorIndex() const noexcept { + return errorIndex; + } + + const char* what() const noexcept override { + static std::string fullMsg; + fullMsg = std::string(IoTDBException::what()) + + "\nParsed data: " + parsedString + + "\nError index: " + std::to_string(errorIndex); + return fullMsg.c_str(); + } +}; + +class IoTDBConnectionException : public IoTDBException { +public: + IoTDBConnectionException() { + } + + explicit IoTDBConnectionException(const char* m) : IoTDBException(m) { + } + + explicit IoTDBConnectionException(const std::string& m) : IoTDBException(m) { + } +}; + +class ExecutionException : public IoTDBException { +public: + ExecutionException() { + } + + explicit ExecutionException(const char* m) : IoTDBException(m) { + } + + explicit ExecutionException(const std::string& m) : IoTDBException(m) { + } + + explicit ExecutionException(const std::string& m, const TSStatus& tsStatus) : IoTDBException(m), status(tsStatus) { + } + + TSStatus status; +}; + +class BatchExecutionException : public IoTDBException { +public: + BatchExecutionException() { + } + + explicit BatchExecutionException(const char* m) : IoTDBException(m) { + } + + explicit BatchExecutionException(const std::string& m) : IoTDBException(m) { + } + + explicit BatchExecutionException(const std::vector& statusList) : statusList(statusList) { + } + + BatchExecutionException(const std::string& m, const std::vector& statusList) : IoTDBException(m), + statusList(statusList) { + } + + std::vector statusList; +}; + +class RedirectException : public IoTDBException { +public: + RedirectException() { + } + + explicit RedirectException(const char* m) : IoTDBException(m) { + } + + explicit RedirectException(const std::string& m) : IoTDBException(m) { + } + + RedirectException(const std::string& m, const TEndPoint& endPoint) : IoTDBException(m), endPoint(endPoint) { + } + + RedirectException(const std::string& m, const map& deviceEndPointMap) : IoTDBException(m), + deviceEndPointMap(deviceEndPointMap) { + } + + RedirectException(const std::string& m, const vector& endPointList) : IoTDBException(m), + endPointList(endPointList) { + } + + TEndPoint endPoint; + map deviceEndPointMap; + vector endPointList; +}; + +class UnSupportedDataTypeException : public IoTDBException { +public: + UnSupportedDataTypeException() { + } + + explicit UnSupportedDataTypeException(const char* m) : IoTDBException(m) { + } + + explicit UnSupportedDataTypeException(const std::string& m) : IoTDBException("UnSupported dataType: " + m) { + } +}; + +class SchemaNotFoundException : public IoTDBException { +public: + SchemaNotFoundException() { + } + + explicit SchemaNotFoundException(const char* m) : IoTDBException(m) { + } + + explicit SchemaNotFoundException(const std::string& m) : IoTDBException(m) { + } +}; + +class StatementExecutionException : public IoTDBException { +public: + StatementExecutionException() { + } + + explicit StatementExecutionException(const char* m) : IoTDBException(m) { + } + + explicit StatementExecutionException(const std::string& m) : IoTDBException(m) { + } +}; + +enum LogLevelType { + LEVEL_DEBUG = 0, + LEVEL_INFO, + LEVEL_WARN, + LEVEL_ERROR +}; + +extern LogLevelType LOG_LEVEL; + +#define log_debug(fmt,...) do {if(LOG_LEVEL <= LEVEL_DEBUG) {string s=string("[DEBUG] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) +#define log_info(fmt,...) do {if(LOG_LEVEL <= LEVEL_INFO) {string s=string("[INFO] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) +#define log_warn(fmt,...) do {if(LOG_LEVEL <= LEVEL_WARN) {string s=string("[WARN] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) +#define log_error(fmt,...) do {if(LOG_LEVEL <= LEVEL_ERROR) {string s=string("[ERROR] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) + +class RpcUtils { +public: + std::shared_ptr SUCCESS_STATUS; + + RpcUtils() { + SUCCESS_STATUS = std::make_shared(); + SUCCESS_STATUS->__set_code(TSStatusCode::SUCCESS_STATUS); + } + + static void verifySuccess(const TSStatus& status); + + static void verifySuccessWithRedirection(const TSStatus& status); + + static void verifySuccessWithRedirectionForMultiDevices(const TSStatus& status, vector devices); + + static void verifySuccess(const std::vector& statuses); + + static TSStatus getStatus(TSStatusCode::TSStatusCode tsStatusCode); + + static TSStatus getStatus(int code, const std::string& message); + + static std::shared_ptr getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode); + + static std::shared_ptr + getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, const std::string& message); + + static std::shared_ptr getTSExecuteStatementResp(const TSStatus& status); + + static std::shared_ptr getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode); + + static std::shared_ptr + getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, const std::string& appendMessage); + + static std::shared_ptr getTSFetchResultsResp(const TSStatus& status); +}; + + +#endif diff --git a/iotdb-client/client-cpp/src/main/DeviceID.h b/iotdb-client/client-cpp/src/main/DeviceID.h new file mode 100644 index 0000000000000..df2682cd5199e --- /dev/null +++ b/iotdb-client/client-cpp/src/main/DeviceID.h @@ -0,0 +1,161 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_DEVICEID_H +#define IOTDB_DEVICEID_H + +#include +#include +#include +#include +#include + +namespace storage { + +static const int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3; +static const std::string PATH_SEPARATOR = "."; + +class IDeviceID { +public: + virtual ~IDeviceID() = default; + virtual std::string get_table_name() { return ""; } + virtual int segment_num() { return 0; } + virtual const std::vector& get_segments() const { + return empty_segments_; + } + virtual std::string get_device_name() const { return ""; }; + virtual bool operator<(const IDeviceID& other) { return 0; } + virtual bool operator==(const IDeviceID& other) { return false; } + virtual bool operator!=(const IDeviceID& other) { return false; } + +protected: + IDeviceID() : empty_segments_() {} + +private: + const std::vector empty_segments_; +}; + +struct IDeviceIDComparator { + bool operator()(const std::shared_ptr& lhs, + const std::shared_ptr& rhs) const { + return *lhs < *rhs; + } +}; + +class StringArrayDeviceID : public IDeviceID { +public: + explicit StringArrayDeviceID(const std::vector& segments) + : segments_(formalize(segments)) {} + + explicit StringArrayDeviceID() : segments_() {} + + ~StringArrayDeviceID() override = default; + + std::string get_device_name() const override { + return segments_.empty() ? "" : std::accumulate(std::next(segments_.begin()), segments_.end(), + segments_.front(), + [](std::string a, const std::string& b) { + return std::move(a) + "." + b; + }); + }; + + std::string get_table_name() override { + return segments_.empty() ? "" : segments_[0]; + } + + int segment_num() override { return static_cast(segments_.size()); } + + const std::vector& get_segments() const override { + return segments_; + } + + bool operator<(const IDeviceID& other) override { + auto other_segments = other.get_segments(); + return std::lexicographical_compare(segments_.begin(), segments_.end(), + other_segments.begin(), + other_segments.end()); + } + + bool operator==(const IDeviceID& other) override { + auto other_segments = other.get_segments(); + return (segments_.size() == other_segments.size()) && + std::equal(segments_.begin(), segments_.end(), + other_segments.begin()); + } + + bool operator!=(const IDeviceID& other) override { + return !(*this == other); + } + +private: + std::vector segments_; + + std::vector formalize( + const std::vector& segments) { + auto it = + std::find_if(segments.rbegin(), segments.rend(), + [](const std::string& seg) { return !seg.empty(); }); + return std::vector(segments.begin(), it.base()); + } + + std::vector split_device_id_string( + const std::vector& splits) { + size_t segment_cnt = splits.size(); + std::vector final_segments; + + if (segment_cnt == 0) { + return final_segments; + } + + if (segment_cnt == 1) { + // "root" -> {"root"} + final_segments.push_back(splits[0]); + } else if (segment_cnt < static_cast( + DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1)) { + // "root.a" -> {"root", "a"} + // "root.a.b" -> {"root.a", "b"} + std::string table_name = std::accumulate( + splits.begin(), splits.end() - 1, std::string(), + [](const std::string& a, const std::string& b) { + return a.empty() ? b : a + PATH_SEPARATOR + b; + }); + final_segments.push_back(table_name); + final_segments.push_back(splits.back()); + } else { + // "root.a.b.c" -> {"root.a.b", "c"} + // "root.a.b.c.d" -> {"root.a.b", "c", "d"} + std::string table_name = std::accumulate( + splits.begin(), + splits.begin() + DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, + std::string(), [](const std::string& a, const std::string& b) { + return a.empty() ? b : a + PATH_SEPARATOR + b; + }); + + final_segments.emplace_back(std::move(table_name)); + final_segments.insert( + final_segments.end(), + splits.begin() + DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, + splits.end()); + } + + return final_segments; + } +}; +} + +#endif \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/NodesSupplier.cpp b/iotdb-client/client-cpp/src/main/NodesSupplier.cpp new file mode 100644 index 0000000000000..932ba0c17fa12 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/NodesSupplier.cpp @@ -0,0 +1,221 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "NodesSupplier.h" +#include "Session.h" +#include +#include +#include + +const std::string NodesSupplier::SHOW_DATA_NODES_COMMAND = "SHOW DATANODES"; +const std::string NodesSupplier::STATUS_COLUMN_NAME = "Status"; +const std::string NodesSupplier::IP_COLUMN_NAME = "RpcAddress"; +const std::string NodesSupplier::PORT_COLUMN_NAME = "RpcPort"; +const std::string NodesSupplier::REMOVING_STATUS = "Removing"; + +const int64_t NodesSupplier::TIMEOUT_IN_MS = 60000; +const int NodesSupplier::FETCH_SIZE = 10000; +const int NodesSupplier::THRIFT_DEFAULT_BUFFER_SIZE = 4096; +const int NodesSupplier::THRIFT_MAX_FRAME_SIZE = 1048576; +const int NodesSupplier::CONNECTION_TIMEOUT_IN_MS = 1000; + +TEndPoint RoundRobinPolicy::select(const std::vector& nodes) { + static std::atomic_uint index{0}; + + if (nodes.empty()) { + throw IoTDBException("No available nodes"); + } + + return nodes[index++ % nodes.size()]; +} + +StaticNodesSupplier::StaticNodesSupplier(const std::vector& nodes, + NodeSelectionPolicy policy) + : availableNodes_(nodes), policy_(std::move(policy)) {} + +boost::optional StaticNodesSupplier::getQueryEndPoint() { + try { + if (availableNodes_.empty()) { + return boost::none; + } + return policy_(availableNodes_); + } catch (const IoTDBException& e) { + return boost::none; + } +} + +std::vector StaticNodesSupplier::getEndPointList() { + return availableNodes_; +} + +StaticNodesSupplier::~StaticNodesSupplier() = default; + +std::shared_ptr NodesSupplier::create( + std::vector endpoints, + std::string userName, std::string password, std::string zoneId, + int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, + int32_t connectionTimeoutInMs, bool useSSL, bool enableRPCCompression, + std::string version, std::chrono::milliseconds refreshInterval, + NodeSelectionPolicy policy) { + if (endpoints.empty()) { + return nullptr; + } + auto supplier = std::make_shared( + userName, password, zoneId, thriftDefaultBufferSize, + thriftMaxFrameSize, connectionTimeoutInMs, useSSL, + enableRPCCompression, version, std::move(endpoints), std::move(policy) + ); + supplier->startBackgroundRefresh(refreshInterval); + return supplier; +} + +NodesSupplier::NodesSupplier( + std::string userName, std::string password, const std::string& zoneId, + int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, + int32_t connectionTimeoutInMs, bool useSSL, bool enableRPCCompression, + std::string version, std::vector endpoints, NodeSelectionPolicy policy) : userName(std::move(userName)), password(std::move(password)), zoneId(zoneId), + thriftDefaultBufferSize(thriftDefaultBufferSize), thriftMaxFrameSize(thriftMaxFrameSize), + connectionTimeoutInMs(connectionTimeoutInMs), useSSL(useSSL), enableRPCCompression(enableRPCCompression), version(version), endpoints(std::move(endpoints)), + selectionPolicy(std::move(policy)) { + deduplicateEndpoints(); +} + +std::vector NodesSupplier::getEndPointList() { + std::lock_guard lock(mutex); + return endpoints; +} + +TEndPoint NodesSupplier::selectQueryEndpoint() { + std::lock_guard lock(mutex); + try { + return selectionPolicy(endpoints); + } catch (const std::exception& e) { + log_error("NodesSupplier::selectQueryEndpoint exception: %s", e.what()); + throw IoTDBException("NodesSupplier::selectQueryEndpoint exception, " + std::string(e.what())); + } +} + +boost::optional NodesSupplier::getQueryEndPoint() { + try { + return selectQueryEndpoint(); + } catch (const IoTDBException& e) { + return boost::none; + } +} + +NodesSupplier::~NodesSupplier() { + stopBackgroundRefresh(); + client->close(); +} + +void NodesSupplier::deduplicateEndpoints() { + std::vector uniqueEndpoints; + uniqueEndpoints.reserve(endpoints.size()); + for (const auto& endpoint : endpoints) { + if (std::find(uniqueEndpoints.begin(), uniqueEndpoints.end(), endpoint) == uniqueEndpoints.end()) { + uniqueEndpoints.push_back(endpoint); + } + } + endpoints = std::move(uniqueEndpoints); +} + +void NodesSupplier::startBackgroundRefresh(std::chrono::milliseconds interval) { + isRunning = true; + refreshThread = std::thread([this, interval] { + while (isRunning) { + refreshEndpointList(); + std::unique_lock cvLock(this->mutex); + refreshCondition.wait_for(cvLock, interval, [this]() { + return !isRunning.load(); + }); + } + }); +} + +std::vector NodesSupplier::fetchLatestEndpoints() { + try { + if (client == nullptr) { + client = std::make_shared(selectionPolicy(endpoints)); + client->init(userName, password, enableRPCCompression, zoneId, version); + } + + auto sessionDataSet = client->executeQueryStatement(SHOW_DATA_NODES_COMMAND); + + uint32_t columnAddrIdx = -1, columnPortIdx = -1, columnStatusIdx = -1; + auto columnNames = sessionDataSet->getColumnNames(); + for (uint32_t i = 0; i < columnNames.size(); i++) { + if (columnNames[i] == IP_COLUMN_NAME) { + columnAddrIdx = i; + } else if (columnNames[i] == PORT_COLUMN_NAME) { + columnPortIdx = i; + } else if (columnNames[i] == STATUS_COLUMN_NAME) { + columnStatusIdx = i; + } + } + + if (columnAddrIdx == -1 || columnPortIdx == -1 || columnStatusIdx == -1) { + throw IoTDBException("Required columns not found in query result."); + } + + std::vector ret; + while (sessionDataSet->hasNext()) { + RowRecord* record = sessionDataSet->next(); + std::string ip = record->fields.at(columnAddrIdx).stringV; + int32_t port = record->fields.at(columnPortIdx).intV; + std::string status = record->fields.at(columnStatusIdx).stringV; + + if (ip == "0.0.0.0" || status == REMOVING_STATUS) { + log_warn("Skipping invalid node: " + ip + ":" + to_string(port)); + continue; + } + TEndPoint endpoint; + endpoint.ip = ip; + endpoint.port = port; + ret.emplace_back(endpoint); + } + + return ret; + } catch (const IoTDBException& e) { + client.reset(); + throw IoTDBException(std::string("NodesSupplier::fetchLatestEndpoints failed: ") + e.what()); + } +} + +void NodesSupplier::refreshEndpointList() { + try { + auto newEndpoints = fetchLatestEndpoints(); + if (newEndpoints.empty()) { + return; + } + + std::lock_guard lock(mutex); + endpoints.swap(newEndpoints); + deduplicateEndpoints(); + } catch (const IoTDBException& e) { + log_error(std::string("NodesSupplier::refreshEndpointList failed: ") + e.what()); + } +} + +void NodesSupplier::stopBackgroundRefresh() noexcept { + if (isRunning.exchange(false)) { + refreshCondition.notify_all(); + if (refreshThread.joinable()) { + refreshThread.join(); + } + } +} \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/NodesSupplier.h b/iotdb-client/client-cpp/src/main/NodesSupplier.h new file mode 100644 index 0000000000000..27965afa9684d --- /dev/null +++ b/iotdb-client/client-cpp/src/main/NodesSupplier.h @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_NODES_SUPPLIER_H +#define IOTDB_NODES_SUPPLIER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ThriftConnection.h" + +class TEndPoint; + +class RoundRobinPolicy { +public: + static TEndPoint select(const std::vector& nodes); +}; + +class INodesSupplier { +public: + virtual ~INodesSupplier() = default; + virtual boost::optional getQueryEndPoint() = 0; + virtual std::vector getEndPointList() = 0; + using NodeSelectionPolicy = std::function&)>; +}; + +class StaticNodesSupplier : public INodesSupplier { +public: + explicit StaticNodesSupplier(const std::vector& nodes, + NodeSelectionPolicy policy = RoundRobinPolicy::select); + + boost::optional getQueryEndPoint() override; + + std::vector getEndPointList() override; + + ~StaticNodesSupplier() override; + +private: + const std::vector availableNodes_; + NodeSelectionPolicy policy_; +}; + +class NodesSupplier : public INodesSupplier { +public: + static const std::string SHOW_DATA_NODES_COMMAND; + static const std::string STATUS_COLUMN_NAME; + static const std::string IP_COLUMN_NAME; + static const std::string PORT_COLUMN_NAME; + static const std::string REMOVING_STATUS; + + static const int64_t TIMEOUT_IN_MS; + static const int FETCH_SIZE; + static const int THRIFT_DEFAULT_BUFFER_SIZE; + static const int THRIFT_MAX_FRAME_SIZE; + static const int CONNECTION_TIMEOUT_IN_MS; + + static std::shared_ptr create( + std::vector endpoints, + std::string userName, std::string password, std::string zoneId = "", + int32_t thriftDefaultBufferSize = ThriftConnection::THRIFT_DEFAULT_BUFFER_SIZE, + int32_t thriftMaxFrameSize = ThriftConnection::THRIFT_MAX_FRAME_SIZE, + int32_t connectionTimeoutInMs = ThriftConnection::CONNECTION_TIMEOUT_IN_MS, + bool useSSL = false, bool enableRPCCompression = false, + std::string version = "V_1_0", + std::chrono::milliseconds refreshInterval = std::chrono::milliseconds(TIMEOUT_IN_MS), + NodeSelectionPolicy policy = RoundRobinPolicy::select + ); + + NodesSupplier( + std::string userName, std::string password, const std::string& zoneId, + int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, + int32_t connectionTimeoutInMs, bool useSSL, bool enableRPCCompression, + std::string version, std::vector endpoints, NodeSelectionPolicy policy + ); + std::vector getEndPointList() override; + + boost::optional getQueryEndPoint() override; + + ~NodesSupplier() override; + +private: + std::string userName; + std::string password; + int32_t thriftDefaultBufferSize; + int32_t thriftMaxFrameSize; + int32_t connectionTimeoutInMs; + bool useSSL; + bool enableRPCCompression; + std::string version; + std::string zoneId; + + std::mutex mutex; + std::vector endpoints; + NodeSelectionPolicy selectionPolicy; + + std::atomic isRunning{false}; + std::thread refreshThread; + std::condition_variable refreshCondition; + + std::shared_ptr client; + + void deduplicateEndpoints(); + + void startBackgroundRefresh(std::chrono::milliseconds interval); + + std::vector fetchLatestEndpoints(); + + void refreshEndpointList(); + + TEndPoint selectQueryEndpoint(); + + void stopBackgroundRefresh() noexcept; +}; + +#endif \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/Session.cpp b/iotdb-client/client-cpp/src/main/Session.cpp index 9468f20dc6887..dbba1eb202761 100644 --- a/iotdb-client/client-cpp/src/main/Session.cpp +++ b/iotdb-client/client-cpp/src/main/Session.cpp @@ -21,6 +21,8 @@ #include #include #include +#include +#include "NodesSupplier.h" using namespace std; @@ -33,201 +35,164 @@ static const int64_t QUERY_TIMEOUT_MS = -1; LogLevelType LOG_LEVEL = LEVEL_DEBUG; -TSDataType::TSDataType getTSDataTypeFromString(const string &str) { - // BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, NULLTYPE - if (str == "BOOLEAN") return TSDataType::BOOLEAN; - else if (str == "INT32") return TSDataType::INT32; - else if (str == "INT64") return TSDataType::INT64; - else if (str == "FLOAT") return TSDataType::FLOAT; - else if (str == "DOUBLE") return TSDataType::DOUBLE; - else if (str == "TEXT") return TSDataType::TEXT; - else if (str == "NULLTYPE") return TSDataType::NULLTYPE; - return TSDataType::TEXT; -} - -void RpcUtils::verifySuccess(const TSStatus &status) { - if (status.code == TSStatusCode::MULTIPLE_ERROR) { - verifySuccess(status.subStatus); - return; - } - if (status.code != TSStatusCode::SUCCESS_STATUS - && status.code != TSStatusCode::REDIRECTION_RECOMMEND) { - throw ExecutionException(to_string(status.code) + ": " + status.message, status); - } -} - -void RpcUtils::verifySuccess(const vector &statuses) { - for (const TSStatus &status: statuses) { - if (status.code != TSStatusCode::SUCCESS_STATUS) { - throw BatchExecutionException(status.message, statuses); - } - } -} - -TSStatus RpcUtils::getStatus(TSStatusCode::TSStatusCode tsStatusCode) { - TSStatus status; - status.__set_code(tsStatusCode); - return status; -} - -TSStatus RpcUtils::getStatus(int code, const string &message) { - TSStatus status; - status.__set_code(code); - status.__set_message(message); - return status; -} - -shared_ptr RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode) { - TSStatus status = getStatus(tsStatusCode); - return getTSExecuteStatementResp(status); -} - -shared_ptr -RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, const string &message) { - TSStatus status = getStatus(tsStatusCode, message); - return getTSExecuteStatementResp(status); -} - -shared_ptr RpcUtils::getTSExecuteStatementResp(const TSStatus &status) { - shared_ptr resp(new TSExecuteStatementResp()); - TSStatus tsStatus(status); - resp->status = tsStatus; - return resp; -} - -shared_ptr RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode) { - TSStatus status = getStatus(tsStatusCode); - return getTSFetchResultsResp(status); -} - -shared_ptr -RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, const string &appendMessage) { - TSStatus status = getStatus(tsStatusCode, appendMessage); - return getTSFetchResultsResp(status); -} - -shared_ptr RpcUtils::getTSFetchResultsResp(const TSStatus &status) { - shared_ptr resp(new TSFetchResultsResp()); - TSStatus tsStatus(status); - resp->__set_status(tsStatus); - return resp; +TSDataType::TSDataType getTSDataTypeFromString(const string& str) { + // BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, STRING, BLOB, TIMESTAMP, DATE, NULLTYPE + if (str == "BOOLEAN") { + return TSDataType::BOOLEAN; + } else if (str == "INT32") { + return TSDataType::INT32; + } else if (str == "INT64") { + return TSDataType::INT64; + } else if (str == "FLOAT") { + return TSDataType::FLOAT; + } else if (str == "DOUBLE") { + return TSDataType::DOUBLE; + } else if (str == "TEXT") { + return TSDataType::TEXT; + } else if (str == "TIMESTAMP") { + return TSDataType::TIMESTAMP; + } else if (str == "DATE") { + return TSDataType::DATE; + } else if (str == "BLOB") { + return TSDataType::BLOB; + } else if (str == "STRING") { + return TSDataType::STRING; + } else if (str == "NULLTYPE") { + return TSDataType::NULLTYPE; + } + return TSDataType::INVALID_DATATYPE; } void Tablet::createColumns() { for (size_t i = 0; i < schemas.size(); i++) { TSDataType::TSDataType dataType = schemas[i].second; switch (dataType) { - case TSDataType::BOOLEAN: - values[i] = new bool[maxRowNumber]; - break; - case TSDataType::INT32: - values[i] = new int[maxRowNumber]; - break; - case TSDataType::INT64: - values[i] = new int64_t[maxRowNumber]; - break; - case TSDataType::FLOAT: - values[i] = new float[maxRowNumber]; - break; - case TSDataType::DOUBLE: - values[i] = new double[maxRowNumber]; - break; - case TSDataType::TEXT: - values[i] = new string[maxRowNumber]; - break; - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); + case TSDataType::BOOLEAN: + values[i] = new bool[maxRowNumber]; + break; + case TSDataType::DATE: + values[i] = new boost::gregorian::date[maxRowNumber]; + break; + case TSDataType::INT32: + values[i] = new int[maxRowNumber]; + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + values[i] = new int64_t[maxRowNumber]; + break; + case TSDataType::FLOAT: + values[i] = new float[maxRowNumber]; + break; + case TSDataType::DOUBLE: + values[i] = new double[maxRowNumber]; + break; + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::TEXT: + values[i] = new string[maxRowNumber]; + break; + default: + throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); } } } void Tablet::deleteColumns() { for (size_t i = 0; i < schemas.size(); i++) { + if (!values[i]) continue; TSDataType::TSDataType dataType = schemas[i].second; switch (dataType) { - case TSDataType::BOOLEAN: { - bool* valueBuf = (bool*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::INT32: { - int* valueBuf = (int*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::INT64: { - int64_t* valueBuf = (int64_t*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::FLOAT: { - float* valueBuf = (float*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::DOUBLE: { - double* valueBuf = (double*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::TEXT: { - string* valueBuf = (string*)(values[i]); - delete[] valueBuf; - break; - } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); - } - } -} - -void Tablet::addValue(size_t schemaId, size_t rowIndex, void* value) { - if (schemaId >= schemas.size()) { - char tmpStr[100]; - sprintf(tmpStr, "Tablet::addValue(), schemaId >= schemas.size(). schemaId=%ld, schemas.size()=%ld.", schemaId, schemas.size()); - throw std::out_of_range(tmpStr); - } - - if (rowIndex >= rowSize) { - char tmpStr[100]; - sprintf(tmpStr, "Tablet::addValue(), rowIndex >= rowSize. rowIndex=%ld, rowSize.size()=%ld.", rowIndex, rowSize); - throw std::out_of_range(tmpStr); - } - - TSDataType::TSDataType dataType = schemas[schemaId].second; - switch (dataType) { case TSDataType::BOOLEAN: { - bool* valueBuf = (bool*)(values[schemaId]); - valueBuf[rowIndex] = *((bool*)value); + bool* valueBuf = (bool*)(values[i]); + delete[] valueBuf; break; } case TSDataType::INT32: { - int* valueBuf = (int*)(values[schemaId]); - valueBuf[rowIndex] = *((int*)value); + int* valueBuf = (int*)(values[i]); + delete[] valueBuf; break; } + case TSDataType::DATE: { + boost::gregorian::date* valueBuf = (boost::gregorian::date*)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::TIMESTAMP: case TSDataType::INT64: { - int64_t* valueBuf = (int64_t*)(values[schemaId]); - valueBuf[rowIndex] = *((int64_t*)value); + int64_t* valueBuf = (int64_t*)(values[i]); + delete[] valueBuf; break; } case TSDataType::FLOAT: { - float* valueBuf = (float*)(values[schemaId]); - valueBuf[rowIndex] = *((float*)value); + float* valueBuf = (float*)(values[i]); + delete[] valueBuf; break; } case TSDataType::DOUBLE: { - double* valueBuf = (double*)(values[schemaId]); - valueBuf[rowIndex] = *((double*)value); + double* valueBuf = (double*)(values[i]); + delete[] valueBuf; break; } + case TSDataType::STRING: + case TSDataType::BLOB: case TSDataType::TEXT: { - string* valueBuf = (string*)(values[schemaId]); - valueBuf[rowIndex] = *(string*)value; + string* valueBuf = (string*)(values[i]); + delete[] valueBuf; break; } default: throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); + } + values[i] = nullptr; + } +} + +void Tablet::deepCopyTabletColValue(void* const* srcPtr, void** destPtr, TSDataType::TSDataType type, int maxRowNumber) { + void *src = *srcPtr; + switch (type) { + case TSDataType::BOOLEAN: + *destPtr = new bool[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(bool)); + break; + case TSDataType::INT32: + *destPtr = new int32_t[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(int32_t)); + break; + case TSDataType::INT64: + case TSDataType::TIMESTAMP: + *destPtr = new int64_t[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(int64_t)); + break; + case TSDataType::FLOAT: + *destPtr = new float[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(float)); + break; + case TSDataType::DOUBLE: + *destPtr = new double[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(double)); + break; + case TSDataType::DATE: { + *destPtr = new boost::gregorian::date[maxRowNumber]; + boost::gregorian::date* srcDate = static_cast(src); + boost::gregorian::date* destDate = static_cast(*destPtr); + for (size_t j = 0; j < maxRowNumber; ++j) { + destDate[j] = srcDate[j]; + } + break; + } + case TSDataType::STRING: + case TSDataType::TEXT: + case TSDataType::BLOB: { + *destPtr = new std::string[maxRowNumber]; + std::string* srcStr = static_cast(src); + std::string* destStr = static_cast(*destPtr); + for (size_t j = 0; j < maxRowNumber; ++j) { + destStr[j] = srcStr[j]; + } + break; + } + default: + break; } } @@ -246,32 +211,38 @@ size_t Tablet::getValueByteSize() { size_t valueOccupation = 0; for (size_t i = 0; i < schemas.size(); i++) { switch (schemas[i].second) { - case TSDataType::BOOLEAN: - valueOccupation += rowSize; - break; - case TSDataType::INT32: - valueOccupation += rowSize * 4; - break; - case TSDataType::INT64: - valueOccupation += rowSize * 8; - break; - case TSDataType::FLOAT: - valueOccupation += rowSize * 4; - break; - case TSDataType::DOUBLE: - valueOccupation += rowSize * 8; - break; - case TSDataType::TEXT: { - valueOccupation += rowSize * 4; - string* valueBuf = (string*)(values[i]); - for (size_t j = 0; j < rowSize; j++) { - valueOccupation += valueBuf[j].size(); - } - break; + case TSDataType::BOOLEAN: + valueOccupation += rowSize; + break; + case TSDataType::INT32: + valueOccupation += rowSize * 4; + break; + case TSDataType::DATE: + valueOccupation += rowSize * 4; + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + valueOccupation += rowSize * 8; + break; + case TSDataType::FLOAT: + valueOccupation += rowSize * 4; + break; + case TSDataType::DOUBLE: + valueOccupation += rowSize * 8; + break; + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::TEXT: { + valueOccupation += rowSize * 4; + string* valueBuf = (string*)(values[i]); + for (size_t j = 0; j < rowSize; j++) { + valueOccupation += valueBuf[j].size(); } - default: - throw UnSupportedDataTypeException( - string("Data type ") + to_string(schemas[i].second) + " is not supported."); + break; + } + default: + throw UnSupportedDataTypeException( + string("Data type ") + to_string(schemas[i].second) + " is not supported."); } } return valueOccupation; @@ -281,7 +252,18 @@ void Tablet::setAligned(bool isAligned) { this->isAligned = isAligned; } -string SessionUtils::getTime(const Tablet &tablet) { +std::shared_ptr Tablet::getDeviceID(int row) { + std::vector id_array(idColumnIndexes.size() + 1); + size_t idArrayIdx = 0; + id_array[idArrayIdx++] = this->deviceId; + for (auto idColumnIndex : idColumnIndexes) { + void* strPtr = getValue(idColumnIndex, row, TSDataType::TEXT); + id_array[idArrayIdx++] = *static_cast(strPtr); + } + return std::make_shared(id_array); +} + +string SessionUtils::getTime(const Tablet& tablet) { MyStringBuffer timeBuffer; unsigned int n = 8u * tablet.rowSize; if (n > timeBuffer.str.capacity()) { @@ -294,102 +276,134 @@ string SessionUtils::getTime(const Tablet &tablet) { return timeBuffer.str; } -string SessionUtils::getValue(const Tablet &tablet) { +string SessionUtils::getValue(const Tablet& tablet) { MyStringBuffer valueBuffer; unsigned int n = 8u * tablet.schemas.size() * tablet.rowSize; if (n > valueBuffer.str.capacity()) { valueBuffer.reserve(n); } - for (size_t i = 0; i < tablet.schemas.size(); i++) { TSDataType::TSDataType dataType = tablet.schemas[i].second; const BitMap& bitMap = tablet.bitMaps[i]; - switch (dataType) { - case TSDataType::BOOLEAN: { - bool* valueBuf = (bool*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putBool(valueBuf[index]); - } - else { - valueBuffer.putBool(false); - } + switch (dataType) { + case TSDataType::BOOLEAN: { + bool* valueBuf = (bool*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putBool(valueBuf[index]); + } + else { + valueBuffer.putBool(false); } - break; } - case TSDataType::INT32: { - int* valueBuf = (int*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putInt(valueBuf[index]); - } - else { - valueBuffer.putInt((numeric_limits::min)()); - } + break; + } + case TSDataType::INT32: { + int* valueBuf = (int*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putInt(valueBuf[index]); + } + else { + valueBuffer.putInt((numeric_limits::min)()); } - break; } - case TSDataType::INT64: { - int64_t* valueBuf = (int64_t*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putInt64(valueBuf[index]); - } - else { - valueBuffer.putInt64((numeric_limits::min)()); - } + break; + } + case TSDataType::DATE: { + boost::gregorian::date* valueBuf = (boost::gregorian::date*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putDate(valueBuf[index]); + } + else { + valueBuffer.putInt(EMPTY_DATE_INT); } - break; } - case TSDataType::FLOAT: { - float* valueBuf = (float*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putFloat(valueBuf[index]); - } - else { - valueBuffer.putFloat((numeric_limits::min)()); - } + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + int64_t* valueBuf = (int64_t*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putInt64(valueBuf[index]); + } + else { + valueBuffer.putInt64((numeric_limits::min)()); } - break; } - case TSDataType::DOUBLE: { - double* valueBuf = (double*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putDouble(valueBuf[index]); - } - else { - valueBuffer.putDouble((numeric_limits::min)()); - } + break; + } + case TSDataType::FLOAT: { + float* valueBuf = (float*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putFloat(valueBuf[index]); + } + else { + valueBuffer.putFloat((numeric_limits::min)()); + } + } + break; + } + case TSDataType::DOUBLE: { + double* valueBuf = (double*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putDouble(valueBuf[index]); + } + else { + valueBuffer.putDouble((numeric_limits::min)()); } - break; } - case TSDataType::TEXT: { - string* valueBuf = (string*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { + break; + } + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::TEXT: { + string* valueBuf = (string*)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { valueBuffer.putString(valueBuf[index]); } - break; + else { + valueBuffer.putString(""); + } } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); + break; + } + default: + throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); } } for (size_t i = 0; i < tablet.schemas.size(); i++) { const BitMap& bitMap = tablet.bitMaps[i]; bool columnHasNull = !bitMap.isAllUnmarked(); - valueBuffer.putChar(columnHasNull ? (char) 1 : (char) 0); + valueBuffer.putChar(columnHasNull ? (char)1 : (char)0); if (columnHasNull) { const vector& bytes = bitMap.getByteArray(); - for (const char byte: bytes) { - valueBuffer.putChar(byte); + for (size_t index = 0; index < tablet.rowSize / 8 + 1; index++) { + valueBuffer.putChar(bytes[index]); } } } return valueBuffer.str; } +bool SessionUtils::isTabletContainsSingleDevice(Tablet tablet) { + if (tablet.rowSize == 1) { + return true; + } + auto firstDeviceId = tablet.getDeviceID(0); + for (int i = 1; i < tablet.rowSize; ++i) { + if (*firstDeviceId != *tablet.getDeviceID(i)) { + return false; + } + } + return true; +} + int SessionDataSet::getFetchSize() { return fetchSize; } @@ -421,8 +435,9 @@ bool SessionDataSet::hasNext() { if (!resp.hasResultSet) { return false; - } else { - TSQueryDataSet *tsQueryDataSet = &(resp.queryDataSet); + } + else { + TSQueryDataSet* tsQueryDataSet = &(resp.queryDataSet); tsQueryDataSetTimeBuffer.str = tsQueryDataSet->time; tsQueryDataSetTimeBuffer.pos = 0; @@ -440,13 +455,14 @@ bool SessionDataSet::hasNext() { } rowsIndex = 0; } - } catch (const TTransportException &e) { + } + catch (const TTransportException& e) { log_debug(e.what()); throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { + } catch (const IoTDBException& e) { log_debug(e.what()); throw; - } catch (exception &e) { + } catch (exception& e) { throw IoTDBException(string("Cannot fetch result from server: ") + e.what()); } } @@ -463,54 +479,64 @@ void SessionDataSet::constructOneRow() { Field field; if (duplicateLocation.find(i) != duplicateLocation.end()) { field = outFields[duplicateLocation[i]]; - } else { - MyStringBuffer *bitmapBuffer = bitmapBuffers[loc].get(); + } + else { + MyStringBuffer* bitmapBuffer = bitmapBuffers[loc].get(); // another new 8 row, should move the bitmap buffer position to next byte if (rowsIndex % 8 == 0) { currentBitmap[loc] = bitmapBuffer->getChar(); } if (!isNull(loc, rowsIndex)) { - MyStringBuffer *valueBuffer = valueBuffers[loc].get(); + MyStringBuffer* valueBuffer = valueBuffers[loc].get(); TSDataType::TSDataType dataType = getTSDataTypeFromString(columnTypeList[i]); field.dataType = dataType; switch (dataType) { - case TSDataType::BOOLEAN: { - bool booleanValue = valueBuffer->getBool(); - field.boolV = booleanValue; - break; - } - case TSDataType::INT32: { - int intValue = valueBuffer->getInt(); - field.intV = intValue; - break; - } - case TSDataType::INT64: { - int64_t longValue = valueBuffer->getInt64(); - field.longV = longValue; - break; - } - case TSDataType::FLOAT: { - float floatValue = valueBuffer->getFloat(); - field.floatV = floatValue; - break; - } - case TSDataType::DOUBLE: { - double doubleValue = valueBuffer->getDouble(); - field.doubleV = doubleValue; - break; - } - case TSDataType::TEXT: { - string stringValue = valueBuffer->getString(); - field.stringV = stringValue; - break; - } - default: { - throw UnSupportedDataTypeException( - string("Data type ") + columnTypeList[i] + " is not supported."); - } + case TSDataType::BOOLEAN: { + bool booleanValue = valueBuffer->getBool(); + field.boolV = booleanValue; + break; + } + case TSDataType::INT32: { + int intValue = valueBuffer->getInt(); + field.intV = intValue; + break; + } + case TSDataType::DATE: { + boost::gregorian::date dateValue = valueBuffer->getDate(); + field.dateV = dateValue; + break; } - } else { + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + int64_t longValue = valueBuffer->getInt64(); + field.longV = longValue; + break; + } + case TSDataType::FLOAT: { + float floatValue = valueBuffer->getFloat(); + field.floatV = floatValue; + break; + } + case TSDataType::DOUBLE: { + double doubleValue = valueBuffer->getDouble(); + field.doubleV = doubleValue; + break; + } + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::TEXT: { + string stringValue = valueBuffer->getString(); + field.stringV = stringValue; + break; + } + default: { + throw UnSupportedDataTypeException( + string("Data type ") + columnTypeList[i] + " is not supported."); + } + } + } + else { field.dataType = TSDataType::NULLTYPE; } loc++; @@ -520,7 +546,8 @@ void SessionDataSet::constructOneRow() { if (!this->isIgnoreTimeStamp) { rowRecord = RowRecord(tsQueryDataSetTimeBuffer.getInt64(), outFields); - } else { + } + else { tsQueryDataSetTimeBuffer.getInt64(); rowRecord = RowRecord(outFields); } @@ -533,7 +560,7 @@ bool SessionDataSet::isNull(int index, int rowNum) { return ((flag >> shift) & bitmap) == 0; } -RowRecord *SessionDataSet::next() { +RowRecord* SessionDataSet::next() { if (!hasCachedRecord) { if (!hasNext()) { return nullptr; @@ -558,13 +585,14 @@ void SessionDataSet::closeOperationHandle(bool forceClose) { try { client->closeOperation(tsStatus, closeReq); RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { + } + catch (const TTransportException& e) { log_debug(e.what()); throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { + } catch (const IoTDBException& e) { log_debug(e.what()); throw; - } catch (exception &e) { + } catch (exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } @@ -589,7 +617,7 @@ string Template::serialize() const { alignedPrefix.emplace(""); } - for (const auto &child: children_) { + for (const auto& child : children_) { stack.push(make_pair("", child.second)); } @@ -609,10 +637,11 @@ string Template::serialize() const { if (cur_node_ptr->isAligned()) { alignedPrefix.emplace(fullPath); } - for (const auto &child: cur_node_ptr->getChildren()) { + for (const auto& child : cur_node_ptr->getChildren()) { stack.push(make_pair(fullPath, child.second)); } - } else { + } + else { buffer.putString(prefix); buffer.putBool(alignedPrefix.find(prefix) != alignedPrefix.end()); buffer.concat(cur_node_ptr->serialize()); @@ -628,17 +657,44 @@ string Template::serialize() const { Session::~Session() { try { close(); - } catch (const exception &e) { + } + catch (const exception& e) { log_debug(e.what()); } } +void Session::removeBrokenSessionConnection(shared_ptr sessionConnection) { + if (enableRedirection) { + this->endPointToSessionConnection.erase(sessionConnection->getEndPoint()); + } + + auto it1 = deviceIdToEndpoint.begin(); + while (it1 != deviceIdToEndpoint.end()) { + if (it1->second == sessionConnection->getEndPoint()) { + it1 = deviceIdToEndpoint.erase(it1); + } + else { + ++it1; + } + } + + auto it2 = tableModelDeviceIdToEndpoint.begin(); + while (it2 != tableModelDeviceIdToEndpoint.end()) { + if (it2->second == sessionConnection->getEndPoint()) { + it2 = tableModelDeviceIdToEndpoint.erase(it2); + } + else { + ++it2; + } + } +} + /** * check whether the batch has been sorted * * @return whether the batch has been sorted */ -bool Session::checkSorted(const Tablet &tablet) { +bool Session::checkSorted(const Tablet& tablet) { for (size_t i = 1; i < tablet.rowSize; i++) { if (tablet.timestamps[i] < tablet.timestamps[i - 1]) { return false; @@ -647,7 +703,7 @@ bool Session::checkSorted(const Tablet &tablet) { return true; } -bool Session::checkSorted(const vector ×) { +bool Session::checkSorted(const vector& times) { for (size_t i = 1; i < times.size(); i++) { if (times[i] < times[i - 1]) { return false; @@ -656,7 +712,7 @@ bool Session::checkSorted(const vector ×) { return true; } -template +template std::vector sortList(const std::vector& valueList, const int* index, int indexLength) { std::vector sortedValues(valueList.size()); for (int i = 0; i < indexLength; i++) { @@ -665,7 +721,7 @@ std::vector sortList(const std::vector& valueList, const int* index, int i return sortedValues; } -template +template void sortValuesList(T* valueList, const int* index, size_t indexLength) { T* sortedValues = new T[indexLength]; for (int i = 0; i < indexLength; i++) { @@ -683,7 +739,7 @@ void Session::sortTablet(Tablet& tablet) { * so we can insert continuous data in value list to get a better performance */ // sort to get index, and use index to sort value list - int *index = new int[tablet.rowSize]; + int* index = new int[tablet.rowSize]; for (size_t i = 0; i < tablet.rowSize; i++) { index[i] = i; } @@ -693,39 +749,46 @@ void Session::sortTablet(Tablet& tablet) { for (size_t i = 0; i < tablet.schemas.size(); i++) { TSDataType::TSDataType dataType = tablet.schemas[i].second; switch (dataType) { - case TSDataType::BOOLEAN: { - sortValuesList((bool*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::INT32: { - sortValuesList((int*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::INT64: { - sortValuesList((int64_t*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::FLOAT: { - sortValuesList((float*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::DOUBLE: { - sortValuesList((double*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::TEXT: { - sortValuesList((string*)(tablet.values[i]), index, tablet.rowSize); - break; - } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); + case TSDataType::BOOLEAN: { + sortValuesList((bool*)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::INT32: { + sortValuesList((int*)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::DATE: { + sortValuesList((boost::gregorian::date*)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + sortValuesList((int64_t*)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::FLOAT: { + sortValuesList((float*)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::DOUBLE: { + sortValuesList((double*)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::TEXT: { + sortValuesList((string*)(tablet.values[i]), index, tablet.rowSize); + break; + } + default: + throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); } } delete[] index; } -void Session::sortIndexByTimestamp(int *index, std::vector ×tamps, int length) { +void Session::sortIndexByTimestamp(int* index, std::vector& timestamps, int length) { if (length <= 1) { return; } @@ -737,18 +800,19 @@ void Session::sortIndexByTimestamp(int *index, std::vector ×tamps, /** * Append value into buffer in Big Endian order to comply with IoTDB server */ -void Session::appendValues(string &buffer, const char *value, int size) { +void Session::appendValues(string& buffer, const char* value, int size) { static bool hasCheckedEndianFlag = false; static bool localCpuIsBigEndian = false; if (!hasCheckedEndianFlag) { hasCheckedEndianFlag = true; - int chk = 0x0201; //used to distinguish CPU's type (BigEndian or LittleEndian) - localCpuIsBigEndian = (0x01 != *(char *) (&chk)); + int chk = 0x0201; //used to distinguish CPU's type (BigEndian or LittleEndian) + localCpuIsBigEndian = (0x01 != *(char*)(&chk)); } if (localCpuIsBigEndian) { buffer.append(value, size); - } else { + } + else { for (int i = size - 1; i >= 0; i--) { buffer.append(value + i, 1); } @@ -756,70 +820,86 @@ void Session::appendValues(string &buffer, const char *value, int size) { } void -Session::putValuesIntoBuffer(const vector &types, const vector &values, string &buf) { +Session::putValuesIntoBuffer(const vector& types, const vector& values, string& buf) { + int32_t date; for (size_t i = 0; i < values.size(); i++) { int8_t typeNum = getDataTypeNumber(types[i]); - buf.append((char *) (&typeNum), sizeof(int8_t)); + buf.append((char*)(&typeNum), sizeof(int8_t)); switch (types[i]) { - case TSDataType::BOOLEAN: - buf.append(values[i], 1); - break; - case TSDataType::INT32: - appendValues(buf, values[i], sizeof(int32_t)); - break; - case TSDataType::INT64: - appendValues(buf, values[i], sizeof(int64_t)); - break; - case TSDataType::FLOAT: - appendValues(buf, values[i], sizeof(float)); - break; - case TSDataType::DOUBLE: - appendValues(buf, values[i], sizeof(double)); - break; - case TSDataType::TEXT: { - int32_t len = (uint32_t) strlen(values[i]); - appendValues(buf, (char *) (&len), sizeof(uint32_t)); - // no need to change the byte order of string value - buf.append(values[i], len); - break; - } - case TSDataType::NULLTYPE: - break; - default: - break; - } - } -} - -int8_t Session::getDataTypeNumber(TSDataType::TSDataType type) { - switch (type) { case TSDataType::BOOLEAN: - return 0; + buf.append(values[i], 1); + break; case TSDataType::INT32: - return 1; + appendValues(buf, values[i], sizeof(int32_t)); + break; + case TSDataType::DATE: + date = parseDateExpressionToInt(*(boost::gregorian::date*)values[i]); + appendValues(buf, (char*)&date, sizeof(int32_t)); + break; + case TSDataType::TIMESTAMP: case TSDataType::INT64: - return 2; + appendValues(buf, values[i], sizeof(int64_t)); + break; case TSDataType::FLOAT: - return 3; + appendValues(buf, values[i], sizeof(float)); + break; case TSDataType::DOUBLE: - return 4; - case TSDataType::TEXT: - return 5; + appendValues(buf, values[i], sizeof(double)); + break; + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::TEXT: { + int32_t len = (uint32_t)strlen(values[i]); + appendValues(buf, (char*)(&len), sizeof(uint32_t)); + // no need to change the byte order of string value + buf.append(values[i], len); + break; + } + case TSDataType::NULLTYPE: + break; default: - return -1; + break; + } + } +} + +int8_t Session::getDataTypeNumber(TSDataType::TSDataType type) { + switch (type) { + case TSDataType::BOOLEAN: + return 0; + case TSDataType::INT32: + return 1; + case TSDataType::INT64: + return 2; + case TSDataType::FLOAT: + return 3; + case TSDataType::DOUBLE: + return 4; + case TSDataType::TEXT: + return 5; + case TSDataType::TIMESTAMP: + return 8; + case TSDataType::DATE: + return 9; + case TSDataType::BLOB: + return 10; + case TSDataType::STRING: + return 11; + default: + return -1; } } string Session::getVersionString(Version::Version version) { switch (version) { - case Version::V_0_12: - return "V_0_12"; - case Version::V_0_13: - return "V_0_13"; - case Version::V_1_0: - return "V_1_0"; - default: - return "V_0_12"; + case Version::V_0_12: + return "V_0_12"; + case Version::V_0_13: + return "V_0_13"; + case Version::V_1_0: + return "V_1_0"; + default: + return "V_0_12"; } } @@ -841,6 +921,154 @@ void Session::initZoneId() { zoneId = zoneStr; } +void Session::initNodesSupplier() { + std::vector endPoints; + TEndPoint endPoint; + endPoint.__set_ip(host); + endPoint.__set_port(rpcPort); + endPoints.emplace_back(endPoint); + if (enableAutoFetch) { + nodesSupplier = NodesSupplier::create(endPoints, username, password); + } + else { + nodesSupplier = make_shared(endPoints); + } +} + +void Session::initDefaultSessionConnection() { + defaultEndPoint.__set_ip(host); + defaultEndPoint.__set_port(rpcPort); + defaultSessionConnection = make_shared(this, defaultEndPoint, zoneId, nodesSupplier, fetchSize, + 60, 500, + sqlDialect, database); +} + +void Session::insertStringRecordsWithLeaderCache(vector deviceIds, vector times, + vector> measurementsList, + vector> valuesList, bool isAligned) { + std::unordered_map, TSInsertStringRecordsReq> recordsGroup; + for (int i = 0; i < deviceIds.size(); i++) { + auto connection = getSessionConnection(deviceIds[i]); + if (recordsGroup.find(connection) == recordsGroup.end()) { + TSInsertStringRecordsReq request; + std::vector emptyPrefixPaths; + std::vector> emptyMeasurementsList; + vector> emptyValuesList; + std::vector emptyTimestamps; + request.__set_isAligned(isAligned); + request.__set_prefixPaths(emptyPrefixPaths); + request.__set_timestamps(emptyTimestamps); + request.__set_measurementsList(emptyMeasurementsList); + request.__set_valuesList(emptyValuesList); + recordsGroup.insert(make_pair(connection, request)); + } + TSInsertStringRecordsReq& existingReq = recordsGroup[connection]; + existingReq.prefixPaths.emplace_back(deviceIds[i]); + existingReq.timestamps.emplace_back(times[i]); + existingReq.measurementsList.emplace_back(measurementsList[i]); + existingReq.valuesList.emplace_back(valuesList[i]); + } + std::function, const TSInsertStringRecordsReq&)> consumer = + [](const std::shared_ptr& c, const TSInsertStringRecordsReq& r) { + c->insertStringRecords(r); + }; + if (recordsGroup.size() == 1) { + insertOnce(recordsGroup, consumer); + } + else { + insertByGroup(recordsGroup, consumer); + } +} + +void Session::insertRecordsWithLeaderCache(vector deviceIds, vector times, + vector> measurementsList, + const vector>& typesList, + vector> valuesList, bool isAligned) { + std::unordered_map, TSInsertRecordsReq> recordsGroup; + for (int i = 0; i < deviceIds.size(); i++) { + auto connection = getSessionConnection(deviceIds[i]); + if (recordsGroup.find(connection) == recordsGroup.end()) { + TSInsertRecordsReq request; + std::vector emptyPrefixPaths; + std::vector> emptyMeasurementsList; + std::vector emptyValuesList; + std::vector emptyTimestamps; + request.__set_isAligned(isAligned); + request.__set_prefixPaths(emptyPrefixPaths); + request.__set_timestamps(emptyTimestamps); + request.__set_measurementsList(emptyMeasurementsList); + request.__set_valuesList(emptyValuesList); + recordsGroup.insert(make_pair(connection, request)); + } + TSInsertRecordsReq& existingReq = recordsGroup[connection]; + existingReq.prefixPaths.emplace_back(deviceIds[i]); + existingReq.timestamps.emplace_back(times[i]); + existingReq.measurementsList.emplace_back(measurementsList[i]); + vector bufferList; + string buffer; + putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + existingReq.valuesList.emplace_back(buffer); + recordsGroup[connection] = existingReq; + } + std::function, const TSInsertRecordsReq&)> consumer = + [](const std::shared_ptr& c, const TSInsertRecordsReq& r) { + c->insertRecords(r); + }; + if (recordsGroup.size() == 1) { + insertOnce(recordsGroup, consumer); + } + else { + insertByGroup(recordsGroup, consumer); + } +} + +void Session::insertTabletsWithLeaderCache(unordered_map tablets, bool sorted, bool isAligned) { + std::unordered_map, TSInsertTabletsReq> tabletsGroup; + if (tablets.empty()) { + throw BatchExecutionException("No tablet is inserting!"); + } + for (const auto& item : tablets) { + if (isAligned != item.second->isAligned) { + throw BatchExecutionException("The tablets should be all aligned or non-aligned!"); + } + if (!checkSorted(*(item.second))) { + sortTablet(*(item.second)); + } + auto deviceId = item.first; + auto tablet = item.second; + auto connection = getSessionConnection(deviceId); + auto it = tabletsGroup.find(connection); + if (it == tabletsGroup.end()) { + TSInsertTabletsReq request; + tabletsGroup[connection] = request; + } + TSInsertTabletsReq& existingReq = tabletsGroup[connection]; + existingReq.prefixPaths.emplace_back(tablet->deviceId); + existingReq.timestampsList.emplace_back(move(SessionUtils::getTime(*tablet))); + existingReq.valuesList.emplace_back(move(SessionUtils::getValue(*tablet))); + existingReq.sizeList.emplace_back(tablet->rowSize); + vector dataTypes; + vector measurements; + for (pair schema : tablet->schemas) { + measurements.push_back(schema.first); + dataTypes.push_back(schema.second); + } + existingReq.measurementsList.emplace_back(measurements); + existingReq.typesList.emplace_back(dataTypes); + } + + std::function, const TSInsertTabletsReq&)> consumer = + [](const std::shared_ptr& c, const TSInsertTabletsReq& r) { + c->insertTablets(r); + }; + if (tabletsGroup.size() == 1) { + insertOnce(tabletsGroup, consumer); + } + else { + insertByGroup(tabletsGroup, consumer); + } +} + void Session::open() { open(false, DEFAULT_TIMEOUT_MS); } @@ -854,67 +1082,18 @@ void Session::open(bool enableRPCCompression, int connectionTimeoutInMs) { return; } - shared_ptr socket(new TSocket(host, rpcPort)); - transport = std::make_shared(socket); - socket->setConnTimeout(connectionTimeoutInMs); - if (!transport->isOpen()) { - try { - transport->open(); - } - catch (TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } + try { + initDefaultSessionConnection(); } - if (enableRPCCompression) { - shared_ptr protocol(new TCompactProtocol(transport)); - client = std::make_shared(protocol); - } else { - shared_ptr protocol(new TBinaryProtocol(transport)); - client = std::make_shared(protocol); - } - - std::map configuration; - configuration["version"] = getVersionString(version); - - TSOpenSessionReq openReq; - openReq.__set_username(username); - openReq.__set_password(password); - openReq.__set_zoneId(zoneId); - openReq.__set_configuration(configuration); - - try { - TSOpenSessionResp openResp; - client->openSession(openResp, openReq); - RpcUtils::verifySuccess(openResp.status); - if (protocolVersion != openResp.serverProtocolVersion) { - if (openResp.serverProtocolVersion == 0) {// less than 0.10 - throw logic_error(string("Protocol not supported, Client version is ") + to_string(protocolVersion) + - ", but Server version is " + to_string(openResp.serverProtocolVersion)); - } - } - - sessionId = openResp.sessionId; - statementId = client->requestStatementId(sessionId); - - if (!zoneId.empty()) { - setTimeZone(zoneId); - } else { - zoneId = getTimeZone(); - } - } catch (const TTransportException &e) { + catch (const exception& e) { log_debug(e.what()); - transport->close(); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - transport->close(); - throw; - } catch (const exception &e) { - log_debug(e.what()); - transport->close(); throw IoTDBException(e.what()); } + zoneId = defaultSessionConnection->zoneId; + + if (enableRedirection) { + endPointToSessionConnection.insert(make_pair(defaultEndPoint, defaultSessionConnection)); + } isClosed = false; } @@ -925,313 +1104,273 @@ void Session::close() { return; } isClosed = true; - - bool needThrowException = false; - string errMsg; - try { - TSCloseSessionReq req; - req.__set_sessionId(sessionId); - TSStatus tsStatus; - client->closeSession(tsStatus, req); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception &e) { - log_debug(e.what()); - errMsg = errMsg + "Session::close() client->closeSession() error, maybe remote server is down. " + e.what() + "\n" ; - needThrowException = true; - } - - try { - if (transport->isOpen()) { - transport->close(); - } - } - catch (const exception &e) { - log_debug(e.what()); - errMsg = errMsg + "Session::close() transport->close() error. " + e.what() + "\n" ; - needThrowException = true; - } - - if (needThrowException) { - throw IoTDBException(errMsg); - } } -void Session::insertRecord(const string &deviceId, int64_t time, - const vector &measurements, - const vector &values) { +void Session::insertRecord(const string& deviceId, int64_t time, + const vector& measurements, + const vector& values) { TSInsertStringRecordReq req; - req.__set_sessionId(sessionId); req.__set_prefixPath(deviceId); req.__set_timestamp(time); req.__set_measurements(measurements); req.__set_values(values); req.__set_isAligned(false); - TSStatus respStatus; try { - client->insertStringRecord(respStatus, req); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + getSessionConnection(deviceId)->insertStringRecord(req); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertStringRecord(req); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } } } -void Session::insertRecord(const string &prefixPath, int64_t time, - const vector &measurements, - const vector &types, - const vector &values) { +void Session::insertRecord(const string& deviceId, int64_t time, + const vector& measurements, + const vector& types, + const vector& values) { TSInsertRecordReq req; - req.__set_sessionId(sessionId); - req.__set_prefixPath(prefixPath); + req.__set_prefixPath(deviceId); req.__set_timestamp(time); req.__set_measurements(measurements); string buffer; putValuesIntoBuffer(types, values, buffer); req.__set_values(buffer); req.__set_isAligned(false); - TSStatus respStatus; try { - client->insertRecord(respStatus, req); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + getSessionConnection(deviceId)->insertRecord(req); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertRecord(req); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } } } -void Session::insertAlignedRecord(const string &deviceId, int64_t time, - const vector &measurements, - const vector &values) { +void Session::insertAlignedRecord(const string& deviceId, int64_t time, + const vector& measurements, + const vector& values) { TSInsertStringRecordReq req; - req.__set_sessionId(sessionId); req.__set_prefixPath(deviceId); req.__set_timestamp(time); req.__set_measurements(measurements); req.__set_values(values); req.__set_isAligned(true); - TSStatus respStatus; try { - client->insertStringRecord(respStatus, req); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + getSessionConnection(deviceId)->insertStringRecord(req); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertStringRecord(req); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } } } -void Session::insertAlignedRecord(const string &prefixPath, int64_t time, - const vector &measurements, - const vector &types, - const vector &values) { +void Session::insertAlignedRecord(const string& deviceId, int64_t time, + const vector& measurements, + const vector& types, + const vector& values) { TSInsertRecordReq req; - req.__set_sessionId(sessionId); - req.__set_prefixPath(prefixPath); + req.__set_prefixPath(deviceId); req.__set_timestamp(time); req.__set_measurements(measurements); string buffer; putValuesIntoBuffer(types, values, buffer); req.__set_values(buffer); - req.__set_isAligned(true); - TSStatus respStatus; + req.__set_isAligned(false); try { - client->insertRecord(respStatus, req); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + getSessionConnection(deviceId)->insertRecord(req); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertRecord(req); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } } } -void Session::insertRecords(const vector &deviceIds, - const vector ×, - const vector> &measurementsList, - const vector> &valuesList) { +void Session::insertRecords(const vector& deviceIds, + const vector& times, + const vector>& measurementsList, + const vector>& valuesList) { size_t len = deviceIds.size(); if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); throw exception(e); } - TSInsertStringRecordsReq request; - request.__set_sessionId(sessionId); - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - request.__set_valuesList(valuesList); - request.__set_isAligned(false); - try { - TSStatus respStatus; - client->insertStringRecords(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + if (enableRedirection) { + insertStringRecordsWithLeaderCache(deviceIds, times, measurementsList, valuesList, false); + } + else { + TSInsertStringRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + request.__set_valuesList(valuesList); + request.__set_isAligned(false); + try { + defaultSessionConnection->insertStringRecords(request); + } + catch (RedirectException& e) { + } } } -void Session::insertRecords(const vector &deviceIds, - const vector ×, - const vector> &measurementsList, - const vector> &typesList, - const vector> &valuesList) { +void Session::insertRecords(const vector& deviceIds, + const vector& times, + const vector>& measurementsList, + const vector>& typesList, + const vector>& valuesList) { size_t len = deviceIds.size(); if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); throw exception(e); } - TSInsertRecordsReq request; - request.__set_sessionId(sessionId); - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - vector bufferList; - for (size_t i = 0; i < valuesList.size(); i++) { - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - bufferList.push_back(buffer); - } - request.__set_valuesList(bufferList); - request.__set_isAligned(false); - try { - TSStatus respStatus; - client->insertRecords(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + if (enableRedirection) { + insertRecordsWithLeaderCache(deviceIds, times, measurementsList, typesList, valuesList, false); + } + else { + TSInsertRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + vector bufferList; + for (size_t i = 0; i < valuesList.size(); i++) { + string buffer; + putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + bufferList.push_back(buffer); + } + request.__set_valuesList(bufferList); + request.__set_isAligned(false); + try { + defaultSessionConnection->insertRecords(request); + } + catch (RedirectException& e) { + } } } -void Session::insertAlignedRecords(const vector &deviceIds, - const vector ×, - const vector> &measurementsList, - const vector> &valuesList) { +void Session::insertAlignedRecords(const vector& deviceIds, + const vector& times, + const vector>& measurementsList, + const vector>& valuesList) { size_t len = deviceIds.size(); if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); throw exception(e); } - TSInsertStringRecordsReq request; - request.__set_sessionId(sessionId); - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - request.__set_valuesList(valuesList); - request.__set_isAligned(true); - try { - TSStatus respStatus; - client->insertStringRecords(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + if (enableRedirection) { + insertStringRecordsWithLeaderCache(deviceIds, times, measurementsList, valuesList, true); + } + else { + TSInsertStringRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + request.__set_valuesList(valuesList); + request.__set_isAligned(true); + try { + defaultSessionConnection->insertStringRecords(request); + } + catch (RedirectException& e) { + } } } -void Session::insertAlignedRecords(const vector &deviceIds, - const vector ×, - const vector> &measurementsList, - const vector> &typesList, - const vector> &valuesList) { +void Session::insertAlignedRecords(const vector& deviceIds, + const vector& times, + const vector>& measurementsList, + const vector>& typesList, + const vector>& valuesList) { size_t len = deviceIds.size(); if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); throw exception(e); } - TSInsertRecordsReq request; - request.__set_sessionId(sessionId); - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - vector bufferList; - for (size_t i = 0; i < valuesList.size(); i++) { - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - bufferList.push_back(buffer); - } - request.__set_valuesList(bufferList); - request.__set_isAligned(true); - try { - TSStatus respStatus; - client->insertRecords(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + if (enableRedirection) { + insertRecordsWithLeaderCache(deviceIds, times, measurementsList, typesList, valuesList, true); + } + else { + TSInsertRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + vector bufferList; + for (size_t i = 0; i < valuesList.size(); i++) { + string buffer; + putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + bufferList.push_back(buffer); + } + request.__set_valuesList(bufferList); + request.__set_isAligned(false); + try { + defaultSessionConnection->insertRecords(request); + } + catch (RedirectException& e) { + } } } -void Session::insertRecordsOfOneDevice(const string &deviceId, - vector ×, - vector> &measurementsList, - vector> &typesList, - vector> &valuesList) { +void Session::insertRecordsOfOneDevice(const string& deviceId, + vector& times, + vector>& measurementsList, + vector>& typesList, + vector>& valuesList) { insertRecordsOfOneDevice(deviceId, times, measurementsList, typesList, valuesList, false); } -void Session::insertRecordsOfOneDevice(const string &deviceId, - vector ×, - vector> &measurementsList, - vector> &typesList, - vector> &valuesList, +void Session::insertRecordsOfOneDevice(const string& deviceId, + vector& times, + vector>& measurementsList, + vector>& typesList, + vector>& valuesList, bool sorted) { - if (!checkSorted(times)) { - int *index = new int[times.size()]; + int* index = new int[times.size()]; for (size_t i = 0; i < times.size(); i++) { index[i] = (int)i; } @@ -1244,7 +1383,6 @@ void Session::insertRecordsOfOneDevice(const string &deviceId, delete[] index; } TSInsertRecordsOfOneDeviceReq request; - request.__set_sessionId(sessionId); request.__set_prefixPath(deviceId); request.__set_timestamps(times); request.__set_measurementsList(measurementsList); @@ -1256,40 +1394,43 @@ void Session::insertRecordsOfOneDevice(const string &deviceId, } request.__set_valuesList(bufferList); request.__set_isAligned(false); - + TSStatus respStatus; try { - TSStatus respStatus; - client->insertRecordsOfOneDevice(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + getSessionConnection(deviceId)->insertRecordsOfOneDevice(request); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertRecordsOfOneDevice(request); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } } } -void Session::insertAlignedRecordsOfOneDevice(const string &deviceId, - vector ×, - vector> &measurementsList, - vector> &typesList, - vector> &valuesList) { +void Session::insertAlignedRecordsOfOneDevice(const string& deviceId, + vector& times, + vector>& measurementsList, + vector>& typesList, + vector>& valuesList) { insertAlignedRecordsOfOneDevice(deviceId, times, measurementsList, typesList, valuesList, false); } -void Session::insertAlignedRecordsOfOneDevice(const string &deviceId, - vector ×, - vector> &measurementsList, - vector> &typesList, - vector> &valuesList, +void Session::insertAlignedRecordsOfOneDevice(const string& deviceId, + vector& times, + vector>& measurementsList, + vector>& typesList, + vector>& valuesList, bool sorted) { - if (!checkSorted(times)) { - int *index = new int[times.size()]; + int* index = new int[times.size()]; for (size_t i = 0; i < times.size(); i++) { index[i] = (int)i; } @@ -1302,7 +1443,6 @@ void Session::insertAlignedRecordsOfOneDevice(const string &deviceId, delete[] index; } TSInsertRecordsOfOneDeviceReq request; - request.__set_sessionId(sessionId); request.__set_prefixPath(deviceId); request.__set_timestamps(times); request.__set_measurementsList(measurementsList); @@ -1314,368 +1454,498 @@ void Session::insertAlignedRecordsOfOneDevice(const string &deviceId, } request.__set_valuesList(bufferList); request.__set_isAligned(true); - + TSStatus respStatus; try { - TSStatus respStatus; - client->insertRecordsOfOneDevice(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + getSessionConnection(deviceId)->insertRecordsOfOneDevice(request); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertRecordsOfOneDevice(request); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } } } -void Session::insertTablet(Tablet &tablet) { +void Session::insertTablet(Tablet& tablet) { try { insertTablet(tablet, false); } - catch (const exception &e) { + catch (const exception& e) { log_debug(e.what()); logic_error error(e.what()); throw exception(error); } } -void Session::buildInsertTabletReq(TSInsertTabletReq &request, int64_t sessionId, Tablet &tablet, bool sorted) { +void Session::buildInsertTabletReq(TSInsertTabletReq& request, Tablet& tablet, bool sorted) { if ((!sorted) && !checkSorted(tablet)) { sortTablet(tablet); } - request.__set_sessionId(sessionId); request.prefixPath = tablet.deviceId; request.measurements.reserve(tablet.schemas.size()); request.types.reserve(tablet.schemas.size()); - for (pair schema: tablet.schemas) { + for (pair schema : tablet.schemas) { request.measurements.push_back(schema.first); request.types.push_back(schema.second); } - request.values = move(SessionUtils::getValue(tablet)); request.timestamps = move(SessionUtils::getTime(tablet)); request.__set_size(tablet.rowSize); request.__set_isAligned(tablet.isAligned); } -void Session::insertTablet(const TSInsertTabletReq &request){ +void Session::insertTablet(TSInsertTabletReq request) { + auto deviceId = request.prefixPath; + try { + getSessionConnection(deviceId)->insertTablet(request); + } + catch (RedirectException& e) { + handleRedirection(deviceId, e.endPoint); + } catch (const IoTDBConnectionException& e) { + if (enableRedirection && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection->insertTablet(request); + } + catch (RedirectException& e) { + } + } + else { + throw e; + } + } +} + +void Session::insertTablet(Tablet& tablet, bool sorted) { + TSInsertTabletReq request; + buildInsertTabletReq(request, tablet, sorted); + insertTablet(request); +} + +void Session::insertRelationalTablet(Tablet& tablet, bool sorted) { + std::unordered_map, Tablet> relationalTabletGroup; + if (tableModelDeviceIdToEndpoint.empty()) { + relationalTabletGroup.insert(make_pair(defaultSessionConnection, tablet)); + } + else if (SessionUtils::isTabletContainsSingleDevice(tablet)) { + relationalTabletGroup.insert(make_pair(getSessionConnection(tablet.getDeviceID(0)), tablet)); + } + else { + for (int row = 0; row < tablet.rowSize; row++) { + auto iDeviceID = tablet.getDeviceID(row); + std::shared_ptr connection = getSessionConnection(iDeviceID); + + auto it = relationalTabletGroup.find(connection); + if (it == relationalTabletGroup.end()) { + Tablet newTablet(tablet.deviceId, tablet.schemas, tablet.columnTypes, tablet.rowSize); + it = relationalTabletGroup.insert(std::make_pair(connection, newTablet)).first; + } + + Tablet& currentTablet = it->second; + int rowIndex = currentTablet.rowSize++; + currentTablet.timestamps[rowIndex] = tablet.timestamps[row]; + for (int col = 0; col < tablet.schemas.size(); col++) { + switch (tablet.schemas[col].second) { + case TSDataType::BOOLEAN: + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(bool*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + case TSDataType::INT32: + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(int32_t*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + case TSDataType::INT64: + case TSDataType::TIMESTAMP: + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(int64_t*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + case TSDataType::FLOAT: + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(float*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + case TSDataType::DOUBLE: + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(double*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + case TSDataType::DATE: { + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(boost::gregorian::date*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + } + case TSDataType::STRING: + case TSDataType::TEXT: + case TSDataType::BLOB: { + currentTablet.addValue(tablet.schemas[col].first, rowIndex, + *(string*)tablet.getValue(col, row, tablet.schemas[col].second)); + break; + } + default: + break; + } + + } + } + } + if (relationalTabletGroup.size() == 1) { + insertRelationalTabletOnce(relationalTabletGroup, sorted); + } + else { + insertRelationalTabletByGroup(relationalTabletGroup, sorted); + } +} + +void Session::insertRelationalTablet(Tablet& tablet) { + insertRelationalTablet(tablet, false); +} + +void Session::insertRelationalTabletOnce(const std::unordered_map, Tablet>& + relationalTabletGroup, bool sorted) { + auto iter = relationalTabletGroup.begin(); + auto connection = iter->first; + auto tablet = iter->second; + TSInsertTabletReq request; + buildInsertTabletReq(request, tablet, sorted); + request.__set_writeToTable(true); + std::vector columnCategories; + for (auto& category : tablet.columnTypes) { + columnCategories.push_back(static_cast(category)); + } + request.__set_columnCategories(columnCategories); try { TSStatus respStatus; - client->insertTablet(respStatus, request); + connection->getSessionClient()->insertTablet(respStatus, request); RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { + } + catch (RedirectException& e) { + auto endPointList = e.endPointList; + for (int i = 0; i < endPointList.size(); i++) { + auto deviceID = tablet.getDeviceID(i); + handleRedirection(deviceID, endPointList[i]); + } + } catch (const IoTDBConnectionException& e) { + if (endPointToSessionConnection.size() > 1) { + removeBrokenSessionConnection(connection); + try { + TSStatus respStatus; + defaultSessionConnection->getSessionClient()->insertTablet(respStatus, request); + RpcUtils::verifySuccess(respStatus); + } + catch (RedirectException& e) { + } + } + else { + throw IoTDBConnectionException(e.what()); + } + } catch (const TTransportException& e) { log_debug(e.what()); throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { + } catch (const IoTDBException& e) { log_debug(e.what()); throw; - } catch (const exception &e) { + } catch (const exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } } -void Session::insertTablet(Tablet &tablet, bool sorted) { - TSInsertTabletReq request; - buildInsertTabletReq(request, sessionId, tablet, sorted); - insertTablet(request); +void Session::insertRelationalTabletByGroup(const std::unordered_map, Tablet>& + relationalTabletGroup, bool sorted) { + // Create a vector to store future objects for asynchronous operations + std::vector> futures; + + for (auto iter = relationalTabletGroup.begin(); iter != relationalTabletGroup.end(); iter++) { + auto connection = iter->first; + auto tablet = iter->second; + + // Launch asynchronous task for each tablet insertion + futures.emplace_back(std::async(std::launch::async, [=]() mutable { + TSInsertTabletReq request; + buildInsertTabletReq(request, tablet, sorted); + request.__set_writeToTable(true); + + std::vector columnCategories; + for (auto& category : tablet.columnTypes) { + columnCategories.push_back(static_cast(category)); + } + request.__set_columnCategories(columnCategories); + + try { + TSStatus respStatus; + connection->getSessionClient()->insertTablet(respStatus, request); + RpcUtils::verifySuccess(respStatus); + } + catch (const TTransportException& e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException& e) { + log_debug(e.what()); + throw; + } catch (const exception& e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } + })); + } + + for (auto& f : futures) { + f.get(); + } } -void Session::insertAlignedTablet(Tablet &tablet) { +void Session::insertAlignedTablet(Tablet& tablet) { insertAlignedTablet(tablet, false); } -void Session::insertAlignedTablet(Tablet &tablet, bool sorted) { +void Session::insertAlignedTablet(Tablet& tablet, bool sorted) { tablet.setAligned(true); try { insertTablet(tablet, sorted); } - catch (const exception &e) { + catch (const exception& e) { log_debug(e.what()); logic_error error(e.what()); throw exception(error); } } -void Session::insertTablets(unordered_map &tablets) { +void Session::insertTablets(unordered_map& tablets) { try { insertTablets(tablets, false); } - catch (const exception &e) { + catch (const exception& e) { log_debug(e.what()); logic_error error(e.what()); throw exception(error); } } -void Session::insertTablets(unordered_map &tablets, bool sorted) { - TSInsertTabletsReq request; - request.__set_sessionId(sessionId); +void Session::insertTablets(unordered_map& tablets, bool sorted) { if (tablets.empty()) { throw BatchExecutionException("No tablet is inserting!"); } auto beginIter = tablets.begin(); - bool isFirstTabletAligned = ((*beginIter).second)->isAligned; - for (const auto &item: tablets) { - if (isFirstTabletAligned != item.second->isAligned) { - throw BatchExecutionException("The tablets should be all aligned or non-aligned!"); + bool isAligned = ((*beginIter).second)->isAligned; + if (enableRedirection) { + insertTabletsWithLeaderCache(tablets, sorted, isAligned); + } + else { + TSInsertTabletsReq request; + for (const auto& item : tablets) { + if (isAligned != item.second->isAligned) { + throw BatchExecutionException("The tablets should be all aligned or non-aligned!"); + } + if (!checkSorted(*(item.second))) { + sortTablet(*(item.second)); + } + request.prefixPaths.push_back(item.second->deviceId); + vector measurements; + vector dataTypes; + for (pair schema : item.second->schemas) { + measurements.push_back(schema.first); + dataTypes.push_back(schema.second); + } + request.measurementsList.push_back(measurements); + request.typesList.push_back(dataTypes); + request.timestampsList.push_back(move(SessionUtils::getTime(*(item.second)))); + request.valuesList.push_back(move(SessionUtils::getValue(*(item.second)))); + request.sizeList.push_back(item.second->rowSize); } - if (!checkSorted(*(item.second))) { - sortTablet(*(item.second)); + request.__set_isAligned(isAligned); + try { + TSStatus respStatus; + defaultSessionConnection->insertTablets(request); + RpcUtils::verifySuccess(respStatus); } - request.prefixPaths.push_back(item.second->deviceId); - vector measurements; - vector dataTypes; - for (pair schema: item.second->schemas) { - measurements.push_back(schema.first); - dataTypes.push_back(schema.second); + catch (RedirectException& e) { } - request.measurementsList.push_back(measurements); - request.typesList.push_back(dataTypes); - request.timestampsList.push_back(move(SessionUtils::getTime(*(item.second)))); - request.valuesList.push_back(move(SessionUtils::getValue(*(item.second)))); - request.sizeList.push_back(item.second->rowSize); - } - request.__set_isAligned(isFirstTabletAligned); - try { - TSStatus respStatus; - client->insertTablets(respStatus, request); - RpcUtils::verifySuccess(respStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); } } -void Session::insertAlignedTablets(unordered_map &tablets, bool sorted) { +void Session::insertAlignedTablets(unordered_map& tablets, bool sorted) { for (auto iter = tablets.begin(); iter != tablets.end(); iter++) { iter->second->setAligned(true); } try { insertTablets(tablets, sorted); } - catch (const exception &e) { + catch (const exception& e) { log_debug(e.what()); logic_error error(e.what()); throw exception(error); } } -void Session::testInsertRecord(const string &deviceId, int64_t time, const vector &measurements, - const vector &values) { +void Session::testInsertRecord(const string& deviceId, int64_t time, const vector& measurements, + const vector& values) { TSInsertStringRecordReq req; - req.__set_sessionId(sessionId); req.__set_prefixPath(deviceId); req.__set_timestamp(time); req.__set_measurements(measurements); req.__set_values(values); TSStatus tsStatus; try { - client->insertStringRecord(tsStatus, req); + defaultSessionConnection->testInsertStringRecord(req); RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { + } + catch (const TTransportException& e) { log_debug(e.what()); throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { + } catch (const IoTDBException& e) { log_debug(e.what()); throw; - } catch (const exception &e) { + } catch (const exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } } -void Session::testInsertTablet(const Tablet &tablet) { +void Session::testInsertTablet(const Tablet& tablet) { TSInsertTabletReq request; - request.__set_sessionId(sessionId); request.prefixPath = tablet.deviceId; - for (pair schema: tablet.schemas) { + for (pair schema : tablet.schemas) { request.measurements.push_back(schema.first); request.types.push_back(schema.second); } request.__set_timestamps(move(SessionUtils::getTime(tablet))); request.__set_values(move(SessionUtils::getValue(tablet))); request.__set_size(tablet.rowSize); - try { TSStatus tsStatus; - client->testInsertTablet(tsStatus, request); + defaultSessionConnection->testInsertTablet(request); RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { + } + catch (const TTransportException& e) { log_debug(e.what()); throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { + } catch (const IoTDBException& e) { log_debug(e.what()); throw; - } catch (const exception &e) { + } catch (const exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } } -void Session::testInsertRecords(const vector &deviceIds, - const vector ×, - const vector> &measurementsList, - const vector> &valuesList) { +void Session::testInsertRecords(const vector& deviceIds, + const vector& times, + const vector>& measurementsList, + const vector>& valuesList) { size_t len = deviceIds.size(); if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { logic_error error("deviceIds, times, measurementsList and valuesList's size should be equal"); throw exception(error); } TSInsertStringRecordsReq request; - request.__set_sessionId(sessionId); request.__set_prefixPaths(deviceIds); request.__set_timestamps(times); request.__set_measurementsList(measurementsList); - request.__set_valuesList(valuesList); - - try { - TSStatus tsStatus; - client->insertStringRecords(tsStatus, request); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} - -void Session::deleteTimeseries(const string &path) { - vector paths; - paths.push_back(path); - deleteTimeseries(paths); -} - -void Session::deleteTimeseries(const vector &paths) { - TSStatus tsStatus; + request.__set_valuesList(valuesList); try { - client->deleteTimeseries(tsStatus, sessionId, paths); + TSStatus tsStatus; + defaultSessionConnection->getSessionClient()->insertStringRecords(tsStatus, request); RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { + } + catch (const TTransportException& e) { log_debug(e.what()); throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { + } catch (const IoTDBException& e) { log_debug(e.what()); throw; - } catch (const exception &e) { + } catch (const exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } } -void Session::deleteData(const string &path, int64_t endTime) { +void Session::deleteTimeseries(const string& path) { + vector paths; + paths.push_back(path); + deleteTimeseries(paths); +} + +void Session::deleteTimeseries(const vector& paths) { + defaultSessionConnection->deleteTimeseries(paths); +} + +void Session::deleteData(const string& path, int64_t endTime) { vector paths; paths.push_back(path); deleteData(paths, LONG_LONG_MIN, endTime); } -void Session::deleteData(const vector &paths, int64_t endTime) { +void Session::deleteData(const vector& paths, int64_t endTime) { deleteData(paths, LONG_LONG_MIN, endTime); } -void Session::deleteData(const vector &paths, int64_t startTime, int64_t endTime) { +void Session::deleteData(const vector& paths, int64_t startTime, int64_t endTime) { TSDeleteDataReq req; - req.__set_sessionId(sessionId); req.__set_paths(paths); req.__set_startTime(startTime); req.__set_endTime(endTime); - TSStatus tsStatus; - try { - client->deleteData(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->deleteData(req); } -void Session::setStorageGroup(const string &storageGroupId) { - TSStatus tsStatus; - try { - client->setStorageGroup(tsStatus, sessionId, storageGroupId); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } +void Session::setStorageGroup(const string& storageGroupId) { + defaultSessionConnection->setStorageGroup(storageGroupId); } -void Session::deleteStorageGroup(const string &storageGroup) { +void Session::deleteStorageGroup(const string& storageGroup) { vector storageGroups; storageGroups.push_back(storageGroup); deleteStorageGroups(storageGroups); } -void Session::deleteStorageGroups(const vector &storageGroups) { - TSStatus tsStatus; - try { - client->deleteStorageGroups(tsStatus, sessionId, storageGroups); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } +void Session::deleteStorageGroups(const vector& storageGroups) { + defaultSessionConnection->deleteStorageGroups(storageGroups); } -void Session::createTimeseries(const string &path, +void Session::createDatabase(const string& database) { + this->setStorageGroup(database); +} + +void Session::deleteDatabase(const string& database) { + this->deleteStorageGroups(vector{database}); +} + +void Session::deleteDatabases(const vector& databases) { + this->deleteStorageGroups(databases); +} + +void Session::createTimeseries(const string& path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor) { try { createTimeseries(path, dataType, encoding, compressor, nullptr, nullptr, nullptr, ""); } - catch (const exception &e) { + catch (const exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } } -void Session::createTimeseries(const string &path, +void Session::createTimeseries(const string& path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor, - map *props, - map *tags, - map *attributes, - const string &measurementAlias) { + map* props, + map* tags, + map* attributes, + const string& measurementAlias) { TSCreateTimeseriesReq req; - req.__set_sessionId(sessionId); req.__set_path(path); req.__set_dataType(dataType); req.__set_encoding(encoding); @@ -1693,52 +1963,37 @@ void Session::createTimeseries(const string &path, if (!measurementAlias.empty()) { req.__set_measurementAlias(measurementAlias); } - - TSStatus tsStatus; - try { - client->createTimeseries(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->createTimeseries(req); } -void Session::createMultiTimeseries(const vector &paths, - const vector &dataTypes, - const vector &encodings, - const vector &compressors, - vector> *propsList, - vector> *tagsList, - vector> *attributesList, - vector *measurementAliasList) { +void Session::createMultiTimeseries(const vector& paths, + const vector& dataTypes, + const vector& encodings, + const vector& compressors, + vector>* propsList, + vector>* tagsList, + vector>* attributesList, + vector* measurementAliasList) { TSCreateMultiTimeseriesReq request; - request.__set_sessionId(sessionId); request.__set_paths(paths); vector dataTypesOrdinal; dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType: dataTypes) { + for (TSDataType::TSDataType dataType : dataTypes) { dataTypesOrdinal.push_back(dataType); } request.__set_dataTypes(dataTypesOrdinal); vector encodingsOrdinal; encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding: encodings) { + for (TSEncoding::TSEncoding encoding : encodings) { encodingsOrdinal.push_back(encoding); } request.__set_encodings(encodingsOrdinal); vector compressorsOrdinal; compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor: compressors) { + for (CompressionType::CompressionType compressor : compressors) { compressorsOrdinal.push_back(compressor); } request.__set_compressors(compressorsOrdinal); @@ -1757,348 +2012,287 @@ void Session::createMultiTimeseries(const vector &paths, request.__set_measurementAliasList(*measurementAliasList); } - try { - TSStatus tsStatus; - client->createMultiTimeseries(tsStatus, request); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->createMultiTimeseries(request); } -void Session::createAlignedTimeseries(const std::string &deviceId, - const std::vector &measurements, - const std::vector &dataTypes, - const std::vector &encodings, - const std::vector &compressors) { +void Session::createAlignedTimeseries(const std::string& deviceId, + const std::vector& measurements, + const std::vector& dataTypes, + const std::vector& encodings, + const std::vector& compressors) { TSCreateAlignedTimeseriesReq request; - request.__set_sessionId(sessionId); request.__set_prefixPath(deviceId); request.__set_measurements(measurements); vector dataTypesOrdinal; dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType: dataTypes) { + for (TSDataType::TSDataType dataType : dataTypes) { dataTypesOrdinal.push_back(dataType); } request.__set_dataTypes(dataTypesOrdinal); vector encodingsOrdinal; encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding: encodings) { + for (TSEncoding::TSEncoding encoding : encodings) { encodingsOrdinal.push_back(encoding); } request.__set_encodings(encodingsOrdinal); vector compressorsOrdinal; compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor: compressors) { + for (CompressionType::CompressionType compressor : compressors) { compressorsOrdinal.push_back(compressor); } request.__set_compressors(compressorsOrdinal); - try { - TSStatus tsStatus; - client->createAlignedTimeseries(tsStatus, request); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->createAlignedTimeseries(request); } -bool Session::checkTimeseriesExists(const string &path) { +bool Session::checkTimeseriesExists(const string& path) { try { std::unique_ptr dataset = executeQueryStatement("SHOW TIMESERIES " + path); bool isExisted = dataset->hasNext(); dataset->closeOperationHandle(); return isExisted; } - catch (const exception &e) { + catch (const exception& e) { log_debug(e.what()); throw IoTDBException(e.what()); } } -int64_t Session::getSessionId() { - return sessionId; -} +shared_ptr Session::getQuerySessionConnection() { + auto endPoint = nodesSupplier->getQueryEndPoint(); + if (!endPoint.is_initialized() || endPointToSessionConnection.empty()) { + return defaultSessionConnection; + } -string Session::getTimeZone() { - if (!zoneId.empty()) { - return zoneId; + auto it = endPointToSessionConnection.find(endPoint.value()); + if (it != endPointToSessionConnection.end()) { + return it->second; } - TSGetTimeZoneResp resp; + + shared_ptr newConnection; try { - client->getTimeZone(resp, sessionId); - RpcUtils::verifySuccess(resp.status); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + newConnection = make_shared(this, endPoint.value(), zoneId, nodesSupplier, + fetchSize, 60, 500, sqlDialect, database); + endPointToSessionConnection.emplace(endPoint.value(), newConnection); + return newConnection; } - return resp.timeZone; + catch (exception& e) { + log_debug("Session::getQuerySessionConnection() exception: " + e.what()); + return newConnection; + } +} + +shared_ptr Session::getSessionConnection(std::string deviceId) { + if (!enableRedirection || + deviceIdToEndpoint.find(deviceId) == deviceIdToEndpoint.end() || + endPointToSessionConnection.find(deviceIdToEndpoint[deviceId]) == endPointToSessionConnection.end()) { + return defaultSessionConnection; + } + return endPointToSessionConnection.find(deviceIdToEndpoint[deviceId])->second; +} + +shared_ptr Session::getSessionConnection(std::shared_ptr deviceId) { + if (!enableRedirection || + tableModelDeviceIdToEndpoint.find(deviceId) == tableModelDeviceIdToEndpoint.end() || + endPointToSessionConnection.find(tableModelDeviceIdToEndpoint[deviceId]) == endPointToSessionConnection.end()) { + return defaultSessionConnection; + } + return endPointToSessionConnection.find(tableModelDeviceIdToEndpoint[deviceId])->second; +} + +string Session::getTimeZone() { + auto ret = defaultSessionConnection->getTimeZone(); + return ret.timeZone; } -void Session::setTimeZone(const string &zoneId) { +void Session::setTimeZone(const string& zoneId) { TSSetTimeZoneReq req; - req.__set_sessionId(sessionId); + req.__set_sessionId(defaultSessionConnection->sessionId); req.__set_timeZone(zoneId); - TSStatus tsStatus; - try { - client->setTimeZone(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - this->zoneId = zoneId; - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); + defaultSessionConnection->setTimeZone(req); +} + +unique_ptr Session::executeQueryStatement(const string& sql) { + return executeQueryStatementMayRedirect(sql, QUERY_TIMEOUT_MS); +} + +unique_ptr Session::executeQueryStatement(const string& sql, int64_t timeoutInMs) { + return executeQueryStatementMayRedirect(sql, timeoutInMs); +} + +void Session::handleQueryRedirection(TEndPoint endPoint) { + if (!enableRedirection) return; + shared_ptr newConnection; + auto it = endPointToSessionConnection.find(endPoint); + if (it != endPointToSessionConnection.end()) { + newConnection = it->second; + } + else { + try { + newConnection = make_shared(this, endPoint, zoneId, nodesSupplier, + fetchSize, 60, 500, sqlDialect, database); + + endPointToSessionConnection.emplace(endPoint, newConnection); + } + catch (exception& e) { + throw IoTDBConnectionException(e.what()); + } } + defaultSessionConnection = newConnection; } -unique_ptr Session::executeQueryStatement(const string &sql) { - return executeQueryStatement(sql, QUERY_TIMEOUT_MS); +void Session::handleRedirection(const std::string& deviceId, TEndPoint endPoint) { + if (!enableRedirection) return; + if (endPoint.ip == "127.0.0.1") return; + deviceIdToEndpoint[deviceId] = endPoint; + + shared_ptr newConnection; + auto it = endPointToSessionConnection.find(endPoint); + if (it != endPointToSessionConnection.end()) { + newConnection = it->second; + } + else { + try { + newConnection = make_shared(this, endPoint, zoneId, nodesSupplier, + fetchSize, 60, 500, sqlDialect, database); + endPointToSessionConnection.emplace(endPoint, newConnection); + } + catch (exception& e) { + deviceIdToEndpoint.erase(deviceId); + throw IoTDBConnectionException(e.what()); + } + } } -unique_ptr Session::executeQueryStatement(const string &sql, int64_t timeoutInMs) { - TSExecuteStatementReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_statement(sql); - req.__set_timeout(timeoutInMs); - req.__set_fetchSize(fetchSize); - TSExecuteStatementResp resp; - try { - client->executeStatement(resp, req); - RpcUtils::verifySuccess(resp.status); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - throw IoTDBException(e.what()); +void Session::handleRedirection(const std::shared_ptr& deviceId, TEndPoint endPoint) { + if (!enableRedirection) return; + if (endPoint.ip == "127.0.0.1") return; + tableModelDeviceIdToEndpoint[deviceId] = endPoint; + + shared_ptr newConnection; + auto it = endPointToSessionConnection.find(endPoint); + if (it != endPointToSessionConnection.end()) { + newConnection = it->second; + } + else { + try { + newConnection = make_shared(this, endPoint, zoneId, nodesSupplier, + fetchSize, 60, 500, sqlDialect, database); + endPointToSessionConnection.emplace(endPoint, newConnection); + } + catch (exception& e) { + tableModelDeviceIdToEndpoint.erase(deviceId); + throw IoTDBConnectionException(e.what()); + } } - shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); - return unique_ptr(new SessionDataSet( - sql, resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, resp.queryId, - statementId, client, sessionId, queryDataSet)); } -void Session::executeNonQueryStatement(const string &sql) { - TSExecuteStatementReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_statement(sql); - req.__set_timeout(0); //0 means no timeout. This value keep consistent to JAVA SDK. - TSExecuteStatementResp resp; +std::unique_ptr Session::executeQueryStatementMayRedirect(const std::string& sql, int64_t timeoutInMs) { + auto sessionConnection = getQuerySessionConnection(); + if (!sessionConnection) { + log_warn("Session connection not found"); + return nullptr; + } try { - client->executeUpdateStatement(resp, req); - RpcUtils::verifySuccess(resp.status); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - throw IoTDBException(e.what()); + return sessionConnection->executeQueryStatement(sql, timeoutInMs); + } + catch (RedirectException& e) { + log_warn("Session connection redirect exception: " + e.what()); + handleQueryRedirection(e.endPoint); + try { + return defaultSessionConnection->executeQueryStatement(sql, timeoutInMs); + } + catch (exception& e) { + log_error("Exception while executing redirected query statement: %s", e.what()); + throw ExecutionException(e.what()); + } + } catch (exception& e) { + log_error("Exception while executing query statement: %s", e.what()); + throw e; } } -unique_ptr Session::executeRawDataQuery(const vector &paths, int64_t startTime, int64_t endTime) { - TSRawDataQueryReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_fetchSize(fetchSize); - req.__set_paths(paths); - req.__set_startTime(startTime); - req.__set_endTime(endTime); - TSExecuteStatementResp resp; +void Session::executeNonQueryStatement(const string& sql) { try { - client->executeRawDataQuery(resp, req); - RpcUtils::verifySuccess(resp.status); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { + defaultSessionConnection->executeNonQueryStatement(sql); + } + catch (const exception& e) { throw IoTDBException(e.what()); } - shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); - return unique_ptr( - new SessionDataSet("", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, - resp.queryId, statementId, client, sessionId, queryDataSet)); +} + +unique_ptr +Session::executeRawDataQuery(const vector& paths, int64_t startTime, int64_t endTime) { + return defaultSessionConnection->executeRawDataQuery(paths, startTime, endTime); } -unique_ptr Session::executeLastDataQuery(const vector &paths) { +unique_ptr Session::executeLastDataQuery(const vector& paths) { return executeLastDataQuery(paths, LONG_LONG_MIN); } -unique_ptr Session::executeLastDataQuery(const vector &paths, int64_t lastTime) { - TSLastDataQueryReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_fetchSize(fetchSize); - req.__set_paths(paths); - req.__set_time(lastTime); - TSExecuteStatementResp resp; - try { - client->executeLastDataQuery(resp, req); - RpcUtils::verifySuccess(resp.status); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - throw IoTDBException(e.what()); - } - shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); - return unique_ptr( - new SessionDataSet("", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, - resp.queryId, statementId, client, sessionId, queryDataSet)); +unique_ptr Session::executeLastDataQuery(const vector& paths, int64_t lastTime) { + return defaultSessionConnection->executeLastDataQuery(paths, lastTime); } -void Session::createSchemaTemplate(const Template &templ) { +void Session::createSchemaTemplate(const Template& templ) { TSCreateSchemaTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(templ.getName()); req.__set_serializedTemplate(templ.serialize()); - TSStatus tsStatus; - try { - client->createSchemaTemplate(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->createSchemaTemplate(req); } -void Session::setSchemaTemplate(const string &template_name, const string &prefix_path) { +void Session::setSchemaTemplate(const string& template_name, const string& prefix_path) { TSSetSchemaTemplateReq req; - req.__set_sessionId(sessionId); req.__set_templateName(template_name); req.__set_prefixPath(prefix_path); - TSStatus tsStatus; - try { - client->setSchemaTemplate(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->setSchemaTemplate(req); } -void Session::unsetSchemaTemplate(const string &prefix_path, const string &template_name) { +void Session::unsetSchemaTemplate(const string& prefix_path, const string& template_name) { TSUnsetSchemaTemplateReq req; - req.__set_sessionId(sessionId); req.__set_templateName(template_name); req.__set_prefixPath(prefix_path); - TSStatus tsStatus; - try { - client->unsetSchemaTemplate(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->unsetSchemaTemplate(req); } -void Session::addAlignedMeasurementsInTemplate(const string &template_name, const vector &measurements, - const vector &dataTypes, - const vector &encodings, - const vector &compressors) { +void Session::addAlignedMeasurementsInTemplate(const string& template_name, const vector& measurements, + const vector& dataTypes, + const vector& encodings, + const vector& compressors) { TSAppendSchemaTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_measurements(measurements); req.__set_isAligned(true); vector dataTypesOrdinal; dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType: dataTypes) { + for (TSDataType::TSDataType dataType : dataTypes) { dataTypesOrdinal.push_back(dataType); } req.__set_dataTypes(dataTypesOrdinal); vector encodingsOrdinal; encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding: encodings) { + for (TSEncoding::TSEncoding encoding : encodings) { encodingsOrdinal.push_back(encoding); } req.__set_encodings(encodingsOrdinal); vector compressorsOrdinal; compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor: compressors) { + for (CompressionType::CompressionType compressor : compressors) { compressorsOrdinal.push_back(compressor); } req.__set_compressors(compressorsOrdinal); - TSStatus tsStatus; - try { - client->appendSchemaTemplate(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->appendSchemaTemplate(req); } -void Session::addAlignedMeasurementsInTemplate(const string &template_name, const string &measurement, +void Session::addAlignedMeasurementsInTemplate(const string& template_name, const string& measurement, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor) { vector measurements(1, measurement); @@ -2108,54 +2302,40 @@ void Session::addAlignedMeasurementsInTemplate(const string &template_name, cons addAlignedMeasurementsInTemplate(template_name, measurements, dataTypes, encodings, compressors); } -void Session::addUnalignedMeasurementsInTemplate(const string &template_name, const vector &measurements, - const vector &dataTypes, - const vector &encodings, - const vector &compressors) { +void Session::addUnalignedMeasurementsInTemplate(const string& template_name, const vector& measurements, + const vector& dataTypes, + const vector& encodings, + const vector& compressors) { TSAppendSchemaTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_measurements(measurements); req.__set_isAligned(false); vector dataTypesOrdinal; dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType: dataTypes) { + for (TSDataType::TSDataType dataType : dataTypes) { dataTypesOrdinal.push_back(dataType); } req.__set_dataTypes(dataTypesOrdinal); vector encodingsOrdinal; encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding: encodings) { + for (TSEncoding::TSEncoding encoding : encodings) { encodingsOrdinal.push_back(encoding); } req.__set_encodings(encodingsOrdinal); vector compressorsOrdinal; compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor: compressors) { + for (CompressionType::CompressionType compressor : compressors) { compressorsOrdinal.push_back(compressor); } req.__set_compressors(compressorsOrdinal); - TSStatus tsStatus; - try { - client->appendSchemaTemplate(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->appendSchemaTemplate(req); } -void Session::addUnalignedMeasurementsInTemplate(const string &template_name, const string &measurement, +void Session::addUnalignedMeasurementsInTemplate(const string& template_name, const string& measurement, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor) { vector measurements(1, measurement); @@ -2165,130 +2345,67 @@ void Session::addUnalignedMeasurementsInTemplate(const string &template_name, co addUnalignedMeasurementsInTemplate(template_name, measurements, dataTypes, encodings, compressors); } -void Session::deleteNodeInTemplate(const string &template_name, const string &path) { +void Session::deleteNodeInTemplate(const string& template_name, const string& path) { TSPruneSchemaTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_path(path); - TSStatus tsStatus; - try { - client->pruneSchemaTemplate(tsStatus, req); - RpcUtils::verifySuccess(tsStatus); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException &e) { - log_debug(e.what()); - throw; - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + defaultSessionConnection->pruneSchemaTemplate(req); } -int Session::countMeasurementsInTemplate(const string &template_name) { +int Session::countMeasurementsInTemplate(const string& template_name) { TSQueryTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_queryType(TemplateQueryType::COUNT_MEASUREMENTS); - TSQueryTemplateResp resp; - try { - client->querySchemaTemplate(resp, req); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + TSQueryTemplateResp resp = defaultSessionConnection->querySchemaTemplate(req); return resp.count; } -bool Session::isMeasurementInTemplate(const string &template_name, const string &path) { +bool Session::isMeasurementInTemplate(const string& template_name, const string& path) { TSQueryTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_measurement(path); req.__set_queryType(TemplateQueryType::IS_MEASUREMENT); - TSQueryTemplateResp resp; - try { - client->querySchemaTemplate(resp, req); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + TSQueryTemplateResp resp = defaultSessionConnection->querySchemaTemplate(req); return resp.result; } -bool Session::isPathExistInTemplate(const string &template_name, const string &path) { +bool Session::isPathExistInTemplate(const string& template_name, const string& path) { TSQueryTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_measurement(path); req.__set_queryType(TemplateQueryType::PATH_EXIST); - TSQueryTemplateResp resp; - try { - client->querySchemaTemplate(resp, req); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + TSQueryTemplateResp resp = defaultSessionConnection->querySchemaTemplate(req); return resp.result; } -std::vector Session::showMeasurementsInTemplate(const string &template_name) { +std::vector Session::showMeasurementsInTemplate(const string& template_name) { TSQueryTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_measurement(""); req.__set_queryType(TemplateQueryType::SHOW_MEASUREMENTS); - TSQueryTemplateResp resp; - try { - client->querySchemaTemplate(resp, req); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + TSQueryTemplateResp resp = defaultSessionConnection->querySchemaTemplate(req); return resp.measurements; } -std::vector Session::showMeasurementsInTemplate(const string &template_name, const string &pattern) { +std::vector Session::showMeasurementsInTemplate(const string& template_name, const string& pattern) { TSQueryTemplateReq req; - req.__set_sessionId(sessionId); req.__set_name(template_name); req.__set_measurement(pattern); req.__set_queryType(TemplateQueryType::SHOW_MEASUREMENTS); - TSQueryTemplateResp resp; - try { - client->querySchemaTemplate(resp, req); - } catch (const TTransportException &e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception &e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } + TSQueryTemplateResp resp = defaultSessionConnection->querySchemaTemplate(req); return resp.measurements; } bool Session::checkTemplateExists(const string& template_name) { try { - std::unique_ptr dataset = executeQueryStatement("SHOW NODES IN DEVICE TEMPLATE " + template_name); + std::unique_ptr dataset = executeQueryStatement( + "SHOW NODES IN DEVICE TEMPLATE " + template_name); bool isExisted = dataset->hasNext(); dataset->closeOperationHandle(); return isExisted; } - catch (const exception &e) { - if ( strstr(e.what(), "get template info error") != NULL ) { + catch (const exception& e) { + if (strstr(e.what(), "does not exist") != NULL) { return false; } log_debug(e.what()); diff --git a/iotdb-client/client-cpp/src/main/Session.h b/iotdb-client/client-cpp/src/main/Session.h index 56418ebcec2d3..e00a6835f821a 100644 --- a/iotdb-client/client-cpp/src/main/Session.h +++ b/iotdb-client/client-cpp/src/main/Session.h @@ -29,18 +29,22 @@ #include #include #include -#include -#include #include -#include #include #include +#include +#include #include #include #include #include #include #include "IClientRPCService.h" +#include "NodesSupplier.h" +#include "AbstractSessionBuilder.h" +#include "SessionConnection.h" +#include "DeviceID.h" +#include "Common.h" //== For compatible with Windows OS == #ifndef LONG_LONG_MIN @@ -59,220 +63,6 @@ using ::apache::thrift::transport::TFramedTransport; using ::apache::thrift::TException; -enum LogLevelType { - LEVEL_DEBUG = 0, - LEVEL_INFO, - LEVEL_WARN, - LEVEL_ERROR -}; -extern LogLevelType LOG_LEVEL; - -#define log_debug(fmt,...) do {if(LOG_LEVEL <= LEVEL_DEBUG) {string s=string("[DEBUG] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) -#define log_info(fmt,...) do {if(LOG_LEVEL <= LEVEL_INFO) {string s=string("[INFO] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) -#define log_warn(fmt,...) do {if(LOG_LEVEL <= LEVEL_WARN) {string s=string("[WARN] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) -#define log_error(fmt,...) do {if(LOG_LEVEL <= LEVEL_ERROR) {string s=string("[ERROR] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) - - -class IoTDBException : public std::exception { -public: - IoTDBException() {} - - explicit IoTDBException(const std::string &m) : message(m) {} - - explicit IoTDBException(const char *m) : message(m) {} - - virtual const char *what() const noexcept override { - return message.c_str(); - } -private: - std::string message; -}; - -class IoTDBConnectionException : public IoTDBException { -public: - IoTDBConnectionException() {} - - explicit IoTDBConnectionException(const char *m) : IoTDBException(m) {} - - explicit IoTDBConnectionException(const std::string &m) : IoTDBException(m) {} -}; - -class ExecutionException : public IoTDBException { -public: - ExecutionException() {} - - explicit ExecutionException(const char *m) : IoTDBException(m) {} - - explicit ExecutionException(const std::string &m) : IoTDBException(m) {} - - explicit ExecutionException(const std::string &m, const TSStatus &tsStatus) : IoTDBException(m), status(tsStatus) {} - - TSStatus status; -}; - -class BatchExecutionException : public IoTDBException { -public: - BatchExecutionException() {} - - explicit BatchExecutionException(const char *m) : IoTDBException(m) {} - - explicit BatchExecutionException(const std::string &m) : IoTDBException(m) {} - - explicit BatchExecutionException(const std::vector &statusList) : statusList(statusList) {} - - BatchExecutionException(const std::string &m, const std::vector &statusList) : IoTDBException(m), statusList(statusList) {} - - std::vector statusList; -}; - -class UnSupportedDataTypeException : public IoTDBException { -public: - UnSupportedDataTypeException() {} - - explicit UnSupportedDataTypeException(const char *m) : IoTDBException(m) {} - - explicit UnSupportedDataTypeException(const std::string &m) : IoTDBException("UnSupported dataType: " + m) {} -}; - -namespace Version { - enum Version { - V_0_12, V_0_13, V_1_0 - }; -} - -namespace CompressionType { - enum CompressionType { - UNCOMPRESSED = (char) 0, - SNAPPY = (char) 1, - GZIP = (char) 2, - LZO = (char) 3, - SDT = (char) 4, - PAA = (char) 5, - PLA = (char) 6, - LZ4 = (char) 7, - ZSTD = (char) 8, - LZMA2 = (char) 9, - }; -} - -namespace TSDataType { - enum TSDataType { - BOOLEAN = (char) 0, - INT32 = (char) 1, - INT64 = (char) 2, - FLOAT = (char) 3, - DOUBLE = (char) 4, - TEXT = (char) 5, - VECTOR = (char) 6, - NULLTYPE = (char) 7 - }; -} - -namespace TSEncoding { - enum TSEncoding { - PLAIN = (char) 0, - DICTIONARY = (char) 1, - RLE = (char) 2, - DIFF = (char) 3, - TS_2DIFF = (char) 4, - BITMAP = (char) 5, - GORILLA_V1 = (char) 6, - REGULAR = (char) 7, - GORILLA = (char) 8, - ZIGZAG = (char) 9, - CHIMP = (char) 11, - SPRINTZ = (char) 12, - RLBE = (char) 13 - }; -} - -namespace TSStatusCode { - enum TSStatusCode { - SUCCESS_STATUS = 200, - - // System level - INCOMPATIBLE_VERSION = 201, - CONFIGURATION_ERROR = 202, - START_UP_ERROR = 203, - SHUT_DOWN_ERROR = 204, - - // General Error - UNSUPPORTED_OPERATION = 300, - EXECUTE_STATEMENT_ERROR = 301, - MULTIPLE_ERROR = 302, - ILLEGAL_PARAMETER = 303, - OVERLAP_WITH_EXISTING_TASK = 304, - INTERNAL_SERVER_ERROR = 305, - - // Client, - REDIRECTION_RECOMMEND = 400, - - // Schema Engine - DATABASE_NOT_EXIST = 500, - DATABASE_ALREADY_EXISTS = 501, - SERIES_OVERFLOW = 502, - TIMESERIES_ALREADY_EXIST = 503, - TIMESERIES_IN_BLACK_LIST = 504, - ALIAS_ALREADY_EXIST = 505, - PATH_ALREADY_EXIST = 506, - METADATA_ERROR = 507, - PATH_NOT_EXIST = 508, - ILLEGAL_PATH = 509, - CREATE_TEMPLATE_ERROR = 510, - DUPLICATED_TEMPLATE = 511, - UNDEFINED_TEMPLATE = 512, - TEMPLATE_NOT_SET = 513, - DIFFERENT_TEMPLATE = 514, - TEMPLATE_IS_IN_USE = 515, - TEMPLATE_INCOMPATIBLE = 516, - SEGMENT_NOT_FOUND = 517, - PAGE_OUT_OF_SPACE = 518, - RECORD_DUPLICATED=519, - SEGMENT_OUT_OF_SPACE = 520, - PBTREE_FILE_NOT_EXISTS = 521, - OVERSIZE_RECORD = 522, - PBTREE_FILE_REDO_LOG_BROKEN = 523, - TEMPLATE_NOT_ACTIVATED = 524, - - // Storage Engine - SYSTEM_READ_ONLY = 600, - STORAGE_ENGINE_ERROR = 601, - STORAGE_ENGINE_NOT_READY = 602, - }; -} - -class RpcUtils { -public: - std::shared_ptr SUCCESS_STATUS; - - RpcUtils() { - SUCCESS_STATUS = std::make_shared(); - SUCCESS_STATUS->__set_code(TSStatusCode::SUCCESS_STATUS); - } - - static void verifySuccess(const TSStatus &status); - - static void verifySuccess(const std::vector &statuses); - - static TSStatus getStatus(TSStatusCode::TSStatusCode tsStatusCode); - - static TSStatus getStatus(int code, const std::string &message); - - static std::shared_ptr getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode); - - static std::shared_ptr - getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, const std::string &message); - - static std::shared_ptr getTSExecuteStatementResp(const TSStatus &status); - - static std::shared_ptr getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode); - - static std::shared_ptr - getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, const std::string &appendMessage); - - static std::shared_ptr getTSFetchResultsResp(const TSStatus &status); -}; - // Simulate the ByteBuffer class in Java class MyStringBuffer { public: @@ -298,7 +88,11 @@ class MyStringBuffer { } int getInt() { - return *(int *) getOrderedByte(4); + return *(int*)getOrderedByte(4); + } + + boost::gregorian::date getDate() { + return parseIntToDate(getInt()); } int64_t getInt64() { @@ -312,12 +106,12 @@ class MyStringBuffer { return *(int64_t*)tmp_buf; } #else - return *(int64_t *) getOrderedByte(8); + return *(int64_t*)getOrderedByte(8); #endif } float getFloat() { - return *(float *) getOrderedByte(4); + return *(float*)getOrderedByte(4); } double getDouble() { @@ -331,7 +125,7 @@ class MyStringBuffer { return *(double*)tmp_buf; } #else - return *(double *) getOrderedByte(8); + return *(double*)getOrderedByte(8); #endif } @@ -351,19 +145,23 @@ class MyStringBuffer { } void putInt(int ins) { - putOrderedByte((char *) &ins, 4); + putOrderedByte((char*)&ins, 4); + } + + void putDate(boost::gregorian::date date) { + putInt(parseDateExpressionToInt(date)); } void putInt64(int64_t ins) { - putOrderedByte((char *) &ins, 8); + putOrderedByte((char*)&ins, 8); } void putFloat(float ins) { - putOrderedByte((char *) &ins, 4); + putOrderedByte((char*)&ins, 4); } void putDouble(double ins) { - putOrderedByte((char *) &ins, 8); + putOrderedByte((char*)&ins, 8); } void putChar(char ins) { @@ -375,12 +173,12 @@ class MyStringBuffer { str += tmp; } - void putString(const std::string &ins) { + void putString(const std::string& ins) { putInt((int)(ins.size())); str += ins; } - void concat(const std::string &ins) { + void concat(const std::string& ins) { str.append(ins); } @@ -390,16 +188,17 @@ class MyStringBuffer { private: void checkBigEndian() { - static int chk = 0x0201; //used to distinguish CPU's type (BigEndian or LittleEndian) - isBigEndian = (0x01 != *(char *) (&chk)); + static int chk = 0x0201; //used to distinguish CPU's type (BigEndian or LittleEndian) + isBigEndian = (0x01 != *(char*)(&chk)); } - const char *getOrderedByte(size_t len) { - const char *p = nullptr; + const char* getOrderedByte(size_t len) { + const char* p = nullptr; if (isBigEndian) { p = str.c_str() + pos; - } else { - const char *tmp = str.c_str(); + } + else { + const char* tmp = str.c_str(); for (size_t i = pos; i < pos + len; i++) { numericBuf[pos + len - 1 - i] = tmp[i]; } @@ -409,10 +208,11 @@ class MyStringBuffer { return p; } - void putOrderedByte(char *buf, int len) { + void putOrderedByte(char* buf, int len) { if (isBigEndian) { str.assign(buf, len); - } else { + } + else { for (int i = len - 1; i > -1; i--) { str += buf[i]; } @@ -421,7 +221,7 @@ class MyStringBuffer { private: bool isBigEndian{}; - char numericBuf[8]{}; //only be used by int, long, float, double etc. + char numericBuf[8]{}; //only be used by int, long, float, double etc. }; class BitMap { @@ -443,7 +243,7 @@ class BitMap { if (position >= size) return false; - bits[position >> 3] |= (char) 1 << (position % 8); + bits[position >> 3] |= (char)1 << (position % 8); return true; } @@ -452,18 +252,18 @@ class BitMap { if (position >= size) return false; - bits[position >> 3] &= ~((char) 1 << (position % 8)); + bits[position >> 3] &= ~((char)1 << (position % 8)); return true; } /** mark as 1 at all positions. */ void markAll() { - std::fill(bits.begin(), bits.end(), (char) 0XFF); + std::fill(bits.begin(), bits.end(), (char)0XFF); } /** mark as 0 at all positions. */ void reset() { - std::fill(bits.begin(), bits.end(), (char) 0); + std::fill(bits.begin(), bits.end(), (char)0); } /** returns the value of the bit with the specified index. */ @@ -471,19 +271,19 @@ class BitMap { if (position >= size) return false; - return (bits[position >> 3] & ((char) 1 << (position % 8))) != 0; + return (bits[position >> 3] & ((char)1 << (position % 8))) != 0; } /** whether all bits are zero, i.e., no Null value */ bool isAllUnmarked() const { size_t j; for (j = 0; j < size >> 3; j++) { - if (bits[j] != (char) 0) { + if (bits[j] != (char)0) { return false; } } for (j = 0; j < size % 8; j++) { - if ((bits[size >> 3] & ((char) 1 << j)) != 0) { + if ((bits[size >> 3] & ((char)1 << j)) != 0) { return false; } } @@ -494,12 +294,12 @@ class BitMap { bool isAllMarked() const { size_t j; for (j = 0; j < size >> 3; j++) { - if (bits[j] != (char) 0XFF) { + if (bits[j] != (char)0XFF) { return false; } } for (j = 0; j < size % 8; j++) { - if ((bits[size >> 3] & ((char) 1 << j)) == 0) { + if ((bits[size >> 3] & ((char)1 << j)) == 0) { return false; } } @@ -524,6 +324,7 @@ class Field { TSDataType::TSDataType dataType; bool boolV; int intV; + boost::gregorian::date dateV; int64_t longV; float floatV; double doubleV; @@ -536,6 +337,58 @@ class Field { Field() = default; }; +enum class ColumnCategory { + TAG, + FIELD, + ATTRIBUTE +}; + +template +void safe_cast(const T& value, Target& target) { + /* + Target Allowed Source Types + BOOLEAN BOOLEAN + INT32 INT32 + INT64 INT32 INT64 + FLOAT INT32 FLOAT + DOUBLE INT32 INT64 FLOAT DOUBLE + TEXT TEXT + */ + if (std::is_same::value) { + target = *(Target*)&value; + } + else if (std::is_same::value && std::is_array::value && std::is_same< + char, typename std::remove_extent::type>::value) { + string tmp((const char*)&value); + target = *(Target*)&tmp; + } + else if (std::is_same::value && std::is_same::value) { + int64_t tmp = *(int32_t*)&value; + target = *(Target*)&tmp; + } + else if (std::is_same::value && std::is_same::value) { + float tmp = *(int32_t*)&value; + target = *(Target*)&tmp; + } + else if (std::is_same::value && std::is_same::value) { + double tmp = *(int32_t*)&value; + target = *(Target*)&tmp; + } + else if (std::is_same::value && std::is_same::value) { + double tmp = *(int64_t*)&value; + target = *(Target*)&tmp; + } + else if (std::is_same::value && std::is_same::value) { + double tmp = *(float*)&value; + target = *(Target*)&tmp; + } + else { + throw UnSupportedDataTypeException("Error: Parameter type " + + std::string(typeid(T).name()) + " cannot be converted to DataType" + + std::string(typeid(Target).name())); + } +} + /* * A tablet data of one device, the tablet contains multiple measurements of this device that share * the same time column. @@ -559,13 +412,17 @@ class Tablet { public: std::string deviceId; // deviceId of this tablet - std::vector> schemas; // the list of measurement schemas for creating the tablet - std::vector timestamps; // timestamps in this tablet + std::vector> schemas; + // the list of measurement schemas for creating the tablet + std::map schemaNameIndex; // the map of schema name to index + std::vector columnTypes; // the list of column types (used in table model) + std::vector timestamps; // timestamps in this tablet std::vector values; // each object is a primitive type array, which represents values of one measurement std::vector bitMaps; // each bitmap represents the existence of each value in the current column - size_t rowSize; //the number of rows to include in this tablet - size_t maxRowNumber; // the maximum number of rows for this tablet - bool isAligned; // whether this tablet store data of aligned timeseries or not + size_t rowSize; //the number of rows to include in this tablet + size_t maxRowNumber; // the maximum number of rows for this tablet + bool isAligned; // whether this tablet store data of aligned timeseries or not + std::vector idColumnIndexes; Tablet() = default; @@ -576,9 +433,16 @@ class Tablet { * @param deviceId the name of the device specified to be written in * @param timeseries the list of measurement schemas for creating the tablet */ - Tablet(const std::string &deviceId, - const std::vector> ×eries) - : Tablet(deviceId, timeseries, DEFAULT_ROW_SIZE) {} + Tablet(const std::string& deviceId, + const std::vector>& timeseries) + : Tablet(deviceId, timeseries, DEFAULT_ROW_SIZE) { + } + + Tablet(const std::string& deviceId, + const std::vector>& timeseries, + const std::vector& columnTypes) + : Tablet(deviceId, timeseries, columnTypes, DEFAULT_ROW_SIZE) { + } /** * Return a tablet with the specified number of rows (maxBatchSize). Only @@ -588,33 +452,205 @@ class Tablet { * @param deviceId the name of the device specified to be written in * @param schemas the list of measurement schemas for creating the row * batch + * @param columnTypes the list of column types (used in table model) * @param maxRowNumber the maximum number of rows for this tablet */ - Tablet(const std::string &deviceId, const std::vector> &schemas, + Tablet(const std::string& deviceId, + const std::vector>& schemas, + int maxRowNumber) + : Tablet(deviceId, schemas, std::vector(schemas.size(), ColumnCategory::FIELD), maxRowNumber) { + } + + Tablet(const std::string& deviceId, const std::vector>& schemas, + const std::vector columnTypes, size_t maxRowNumber, bool _isAligned = false) : deviceId(deviceId), schemas(schemas), - maxRowNumber(maxRowNumber), isAligned(_isAligned) { + columnTypes(columnTypes), + maxRowNumber(maxRowNumber), isAligned(_isAligned) { // create timestamp column timestamps.resize(maxRowNumber); // create value columns values.resize(schemas.size()); createColumns(); + // init idColumnIndexs + for (size_t i = 0; i < this->columnTypes.size(); i++) { + if (this->columnTypes[i] == ColumnCategory::TAG) { + idColumnIndexes.push_back(i); + } + } // create bitMaps bitMaps.resize(schemas.size()); for (size_t i = 0; i < schemas.size(); i++) { bitMaps[i].resize(maxRowNumber); } + // create schemaNameIndex + for (size_t i = 0; i < schemas.size(); i++) { + schemaNameIndex[schemas[i].first] = i; + } this->rowSize = 0; } + Tablet(const Tablet& other) + : deviceId(other.deviceId), + schemas(other.schemas), + schemaNameIndex(other.schemaNameIndex), + columnTypes(other.columnTypes), + timestamps(other.timestamps), + maxRowNumber(other.maxRowNumber), + bitMaps(other.bitMaps), + rowSize(other.rowSize), + isAligned(other.isAligned), + idColumnIndexes(other.idColumnIndexes) { + values.resize(other.values.size()); + for (size_t i = 0; i < other.values.size(); ++i) { + if (!other.values[i]) continue; + TSDataType::TSDataType type = schemas[i].second; + deepCopyTabletColValue(&(other.values[i]), &values[i], type, maxRowNumber); + } + } + + Tablet& operator=(const Tablet& other) { + if (this != &other) { + deleteColumns(); + deviceId = other.deviceId; + schemas = other.schemas; + schemaNameIndex = other.schemaNameIndex; + columnTypes = other.columnTypes; + timestamps = other.timestamps; + maxRowNumber = other.maxRowNumber; + rowSize = other.rowSize; + isAligned = other.isAligned; + idColumnIndexes = other.idColumnIndexes; + bitMaps = other.bitMaps; + values.resize(other.values.size()); + for (size_t i = 0; i < other.values.size(); ++i) { + if (!other.values[i]) continue; + TSDataType::TSDataType type = schemas[i].second; + deepCopyTabletColValue(&(other.values[i]), &values[i], type, maxRowNumber); + } + } + return *this; + } + ~Tablet() { try { deleteColumns(); - } catch (exception &e) { + } + catch (exception& e) { log_debug(string("Tablet::~Tablet(), ") + e.what()); } } - void addValue(size_t schemaId, size_t rowIndex, void *value); + void addTimestamp(size_t rowIndex, int64_t timestamp) { + timestamps[rowIndex] = timestamp; + rowSize = max(rowSize, rowIndex + 1); + } + + static void deepCopyTabletColValue(void* const* srcPtr, void** destPtr, + TSDataType::TSDataType type, int maxRowNumber); + + template + void addValue(size_t schemaId, size_t rowIndex, const T& value) { + if (schemaId >= schemas.size()) { + char tmpStr[100]; + sprintf(tmpStr, "Tablet::addValue(), schemaId >= schemas.size(). schemaId=%ld, schemas.size()=%ld.", + schemaId, schemas.size()); + throw std::out_of_range(tmpStr); + } + + if (rowIndex >= rowSize) { + char tmpStr[100]; + sprintf(tmpStr, "Tablet::addValue(), rowIndex >= rowSize. rowIndex=%ld, rowSize.size()=%ld.", rowIndex, + rowSize); + throw std::out_of_range(tmpStr); + } + + TSDataType::TSDataType dataType = schemas[schemaId].second; + switch (dataType) { + case TSDataType::BOOLEAN: { + safe_cast(value, ((bool*)values[schemaId])[rowIndex]); + break; + } + case TSDataType::INT32: { + safe_cast(value, ((int*)values[schemaId])[rowIndex]); + break; + } + case TSDataType::DATE: { + safe_cast(value, ((boost::gregorian::date*)values[schemaId])[rowIndex]); + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + safe_cast(value, ((int64_t*)values[schemaId])[rowIndex]); + break; + } + case TSDataType::FLOAT: { + safe_cast(value, ((float*)values[schemaId])[rowIndex]); + break; + } + case TSDataType::DOUBLE: { + safe_cast(value, ((double*)values[schemaId])[rowIndex]); + break; + } + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::TEXT: { + safe_cast(value, ((string*)values[schemaId])[rowIndex]); + break; + } + default: + throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); + } + } + + template + void addValue(const string& schemaName, size_t rowIndex, const T& value) { + if (schemaNameIndex.find(schemaName) == schemaNameIndex.end()) { + throw SchemaNotFoundException(string("Schema ") + schemaName + " not found."); + } + size_t schemaId = schemaNameIndex[schemaName]; + addValue(schemaId, rowIndex, value); + } + + + void* getValue(size_t schemaId, size_t rowIndex, TSDataType::TSDataType dataType) { + if (schemaId >= schemas.size()) { + throw std::out_of_range("Tablet::getValue schemaId out of range: " + + std::to_string(schemaId)); + } + if (rowIndex >= rowSize) { + throw std::out_of_range("Tablet::getValue rowIndex out of range: " + + std::to_string(rowIndex)); + } + + switch (dataType) { + case TSDataType::BOOLEAN: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::INT32: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::DATE: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::FLOAT: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::DOUBLE: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::TEXT: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + default: + throw UnSupportedDataTypeException("Unsupported data type: " + + std::to_string(dataType)); + } + } + + std::shared_ptr getDeviceID(int i); + + std::vector> getSchemas() const { + return schemas; + } void reset(); // Reset Tablet to the default state - set the rowSize to 0 @@ -627,9 +663,11 @@ class Tablet { class SessionUtils { public: - static std::string getTime(const Tablet &tablet); + static std::string getTime(const Tablet& tablet); - static std::string getValue(const Tablet &tablet); + static std::string getValue(const Tablet& tablet); + + static bool isTabletContainsSingleDevice(Tablet tablet); }; class RowRecord { @@ -641,19 +679,19 @@ class RowRecord { this->timestamp = timestamp; } - RowRecord(int64_t timestamp, const std::vector &fields) - : timestamp(timestamp), fields(fields) { + RowRecord(int64_t timestamp, const std::vector& fields) + : timestamp(timestamp), fields(fields) { } - explicit RowRecord(const std::vector &fields) - : timestamp(-1), fields(fields) { + explicit RowRecord(const std::vector& fields) + : timestamp(-1), fields(fields) { } RowRecord() { this->timestamp = -1; } - void addField(const Field &f) { + void addField(const Field& f) { this->fields.push_back(f); } @@ -669,29 +707,35 @@ class RowRecord { } TSDataType::TSDataType dataType = fields[i].dataType; switch (dataType) { - case TSDataType::BOOLEAN: - ret.append(fields[i].boolV ? "true" : "false"); - break; - case TSDataType::INT32: - ret.append(std::to_string(fields[i].intV)); - break; - case TSDataType::INT64: - ret.append(std::to_string(fields[i].longV)); - break; - case TSDataType::FLOAT: - ret.append(std::to_string(fields[i].floatV)); - break; - case TSDataType::DOUBLE: - ret.append(std::to_string(fields[i].doubleV)); - break; - case TSDataType::TEXT: - ret.append(fields[i].stringV); - break; - case TSDataType::NULLTYPE: - ret.append("NULL"); - break; - default: - break; + case TSDataType::BOOLEAN: + ret.append(fields[i].boolV ? "true" : "false"); + break; + case TSDataType::INT32: + ret.append(std::to_string(fields[i].intV)); + break; + case TSDataType::DATE: + ret.append(boost::gregorian::to_simple_string(fields[i].dateV)); + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + ret.append(std::to_string(fields[i].longV)); + break; + case TSDataType::FLOAT: + ret.append(std::to_string(fields[i].floatV)); + break; + case TSDataType::DOUBLE: + ret.append(std::to_string(fields[i].doubleV)); + break; + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::TEXT: + ret.append(fields[i].stringV); + break; + case TSDataType::NULLTYPE: + ret.append("NULL"); + break; + default: + break; } } ret.append("\n"); @@ -717,7 +761,7 @@ class SessionDataSet { std::unordered_map columnMap; // column size int columnSize = 0; - int columnFieldStartIndex = 0; //Except Timestamp column, 1st field's pos in columnNameList + int columnFieldStartIndex = 0; //Except Timestamp column, 1st field's pos in columnNameList bool isIgnoreTimeStamp = false; int rowsIndex = 0; // used to record the row index in current TSQueryDataSet @@ -726,20 +770,20 @@ class SessionDataSet { std::vector> valueBuffers; std::vector> bitmapBuffers; RowRecord rowRecord; - char *currentBitmap = nullptr; // used to cache the current bitmap for every column + char* currentBitmap = nullptr; // used to cache the current bitmap for every column static const int flag = 0x80; // used to do `or` operation with bitmap to judge whether the value is null bool operationIsOpen = false; public: - SessionDataSet(const std::string &sql, - const std::vector &columnNameList, - const std::vector &columnTypeList, - std::map &columnNameIndexMap, + SessionDataSet(const std::string& sql, + const std::vector& columnNameList, + const std::vector& columnTypeList, + std::map& columnNameIndexMap, bool isIgnoreTimeStamp, int64_t queryId, int64_t statementId, std::shared_ptr client, int64_t sessionId, - const std::shared_ptr &queryDataSet) : tsQueryDataSetTimeBuffer(queryDataSet->time) { + const std::shared_ptr& queryDataSet) : tsQueryDataSetTimeBuffer(queryDataSet->time) { this->sessionId = sessionId; this->sql = sql; this->queryId = queryId; @@ -763,14 +807,16 @@ class SessionDataSet { std::string name = this->columnNameList[i]; if (this->columnMap.find(name) != this->columnMap.end()) { duplicateLocation[i] = columnToFirstIndexMap[name]; - } else { + } + else { columnToFirstIndexMap[name] = i; if (!columnNameIndexMap.empty()) { int valueIndex = columnNameIndexMap[name]; this->columnMap[name] = valueIndex; this->valueBuffers.emplace_back(new MyStringBuffer(queryDataSet->valueList[valueIndex])); this->bitmapBuffers.emplace_back(new MyStringBuffer(queryDataSet->bitmapList[valueIndex])); - } else { + } + else { this->columnMap[name] = deduplicateIdx; this->valueBuffers.emplace_back(new MyStringBuffer(queryDataSet->valueList[deduplicateIdx])); this->bitmapBuffers.emplace_back(new MyStringBuffer(queryDataSet->bitmapList[deduplicateIdx])); @@ -786,7 +832,8 @@ class SessionDataSet { ~SessionDataSet() { try { closeOperationHandle(); - } catch (exception &e) { + } + catch (exception& e) { log_debug(string("SessionDataSet::~SessionDataSet(), ") + e.what()); } @@ -810,20 +857,21 @@ class SessionDataSet { bool isNull(int index, int rowNum); - RowRecord *next(); + RowRecord* next(); void closeOperationHandle(bool forceClose = false); }; class TemplateNode { public: - explicit TemplateNode(const std::string &name) : name_(name) {} + explicit TemplateNode(const std::string& name) : name_(name) { + } - const std::string &getName() const { + const std::string& getName() const { return name_; } - virtual const std::unordered_map> &getChildren() const { + virtual const std::unordered_map>& getChildren() const { throw BatchExecutionException("Should call exact sub class!"); } @@ -843,8 +891,7 @@ class TemplateNode { class MeasurementNode : public TemplateNode { public: - - MeasurementNode(const std::string &name_, TSDataType::TSDataType data_type_, TSEncoding::TSEncoding encoding_, + MeasurementNode(const std::string& name_, TSDataType::TSDataType data_type_, TSEncoding::TSEncoding encoding_, CompressionType::CompressionType compression_type_) : TemplateNode(name_) { this->data_type_ = data_type_; this->encoding_ = encoding_; @@ -877,28 +924,28 @@ class MeasurementNode : public TemplateNode { class InternalNode : public TemplateNode { public: + InternalNode(const std::string& name, bool is_aligned) : TemplateNode(name), is_aligned_(is_aligned) { + } - InternalNode(const std::string &name, bool is_aligned) : TemplateNode(name), is_aligned_(is_aligned) {} - - void addChild(const InternalNode &node) { + void addChild(const InternalNode& node) { if (this->children_.count(node.getName())) { throw BatchExecutionException("Duplicated child of node in template."); } this->children_[node.getName()] = std::make_shared(node); } - void addChild(const MeasurementNode &node) { + void addChild(const MeasurementNode& node) { if (this->children_.count(node.getName())) { throw BatchExecutionException("Duplicated child of node in template."); } this->children_[node.getName()] = std::make_shared(node); } - void deleteChild(const TemplateNode &node) { + void deleteChild(const TemplateNode& node) { this->children_.erase(node.getName()); } - const std::unordered_map> &getChildren() const override { + const std::unordered_map>& getChildren() const override { return children_; } @@ -916,17 +963,17 @@ class InternalNode : public TemplateNode { }; namespace TemplateQueryType { - enum TemplateQueryType { - COUNT_MEASUREMENTS, IS_MEASUREMENT, PATH_EXIST, SHOW_MEASUREMENTS - }; +enum TemplateQueryType { + COUNT_MEASUREMENTS, IS_MEASUREMENT, PATH_EXIST, SHOW_MEASUREMENTS +}; } class Template { public: + Template(const std::string& name, bool is_aligned) : name_(name), is_aligned_(is_aligned) { + } - Template(const std::string &name, bool is_aligned) : name_(name), is_aligned_(is_aligned) {} - - const std::string &getName() const { + const std::string& getName() const { return name_; } @@ -934,14 +981,14 @@ class Template { return is_aligned_; } - void addToTemplate(const InternalNode &child) { + void addToTemplate(const InternalNode& child) { if (this->children_.count(child.getName())) { throw BatchExecutionException("Duplicated child of node in template."); } this->children_[child.getName()] = std::make_shared(child); } - void addToTemplate(const MeasurementNode &child) { + void addToTemplate(const MeasurementNode& child) { if (this->children_.count(child.getName())) { throw BatchExecutionException("Duplicated child of node in template."); } @@ -963,38 +1010,65 @@ class Session { std::string username; std::string password; const TSProtocolVersion::type protocolVersion = TSProtocolVersion::IOTDB_SERVICE_PROTOCOL_V3; - std::shared_ptr client; - std::shared_ptr transport; bool isClosed = true; - int64_t sessionId; - int64_t statementId; std::string zoneId; int fetchSize; const static int DEFAULT_FETCH_SIZE = 10000; const static int DEFAULT_TIMEOUT_MS = 0; Version::Version version; + std::string sqlDialect = "tree"; // default sql dialect + std::string database; + bool enableAutoFetch = true; + bool enableRedirection = true; + std::shared_ptr nodesSupplier; + friend class SessionConnection; + friend class TableSession; + std::shared_ptr defaultSessionConnection; + + TEndPoint defaultEndPoint; + + struct TEndPointHash { + size_t operator()(const TEndPoint& endpoint) const { + return std::hash()(endpoint.ip) ^ std::hash()(endpoint.port); + } + }; + + struct TEndPointEqual { + bool operator()(const TEndPoint& lhs, const TEndPoint& rhs) const { + return lhs.ip == rhs.ip && lhs.port == rhs.port; + } + }; + + using EndPointSessionMap = std::unordered_map< + TEndPoint, shared_ptr, TEndPointHash, TEndPointEqual>; + EndPointSessionMap endPointToSessionConnection; + std::unordered_map deviceIdToEndpoint; + std::unordered_map, TEndPoint> tableModelDeviceIdToEndpoint; private: - static bool checkSorted(const Tablet &tablet); + void removeBrokenSessionConnection(shared_ptr sessionConnection); - static bool checkSorted(const std::vector ×); + static bool checkSorted(const Tablet& tablet); - static void sortTablet(Tablet &tablet); + static bool checkSorted(const std::vector& times); - static void sortIndexByTimestamp(int *index, std::vector ×tamps, int length); + static void sortTablet(Tablet& tablet); - void appendValues(std::string &buffer, const char *value, int size); + static void sortIndexByTimestamp(int* index, std::vector& timestamps, int length); + + void appendValues(std::string& buffer, const char* value, int size); void - putValuesIntoBuffer(const std::vector &types, const std::vector &values, - std::string &buf); + putValuesIntoBuffer(const std::vector& types, const std::vector& values, + std::string& buf); int8_t getDataTypeNumber(TSDataType::TSDataType type); struct TsCompare { - std::vector ×tamps; + std::vector& timestamps; - explicit TsCompare(std::vector &inTimestamps) : timestamps(inTimestamps) {}; + explicit TsCompare(std::vector& inTimestamps) : timestamps(inTimestamps) { + }; bool operator()(int i, int j) { return (timestamps[i] < timestamps[j]); }; }; @@ -1003,25 +1077,78 @@ class Session { void initZoneId(); + void initNodesSupplier(); + + void initDefaultSessionConnection(); + + template + void insertByGroup(std::unordered_map, T>& insertGroup, + InsertConsumer insertConsumer); + + template + void insertOnce(std::unordered_map, T>& insertGroup, + InsertConsumer insertConsumer); + + void insertStringRecordsWithLeaderCache(vector deviceIds, vector times, + vector> measurementsList, vector> valuesList, + bool isAligned); + + void insertRecordsWithLeaderCache(vector deviceIds, vector times, + vector> measurementsList, + const vector>& typesList, + vector> valuesList, bool isAligned); + + void insertTabletsWithLeaderCache(unordered_map tablets, bool sorted, bool isAligned); + + shared_ptr getQuerySessionConnection(); + + shared_ptr getSessionConnection(std::string deviceId); + + shared_ptr getSessionConnection(std::shared_ptr deviceId); + + void handleQueryRedirection(TEndPoint endPoint); + + void handleRedirection(const std::string& deviceId, TEndPoint endPoint); + + void handleRedirection(const std::shared_ptr& deviceId, TEndPoint endPoint); + + void setSqlDialect(const std::string& dialect) { + this->sqlDialect = dialect; + } + + void setDatabase(const std::string& database) { + this->database = database; + } + + string getDatabase() { + return database; + } + + void changeDatabase(string database) { + this->database = database; + } + public: - Session(const std::string &host, int rpcPort) : username("user"), password("password"), version(Version::V_1_0) { + Session(const std::string& host, int rpcPort) : username("root"), password("root"), version(Version::V_1_0) { this->host = host; this->rpcPort = rpcPort; initZoneId(); + initNodesSupplier(); } - Session(const std::string &host, int rpcPort, const std::string &username, const std::string &password) - : fetchSize(DEFAULT_FETCH_SIZE) { + Session(const std::string& host, int rpcPort, const std::string& username, const std::string& password) + : fetchSize(DEFAULT_FETCH_SIZE) { this->host = host; this->rpcPort = rpcPort; this->username = username; this->password = password; this->version = Version::V_1_0; initZoneId(); + initNodesSupplier(); } - Session(const std::string &host, int rpcPort, const std::string &username, const std::string &password, - const std::string &zoneId, int fetchSize = DEFAULT_FETCH_SIZE) { + Session(const std::string& host, int rpcPort, const std::string& username, const std::string& password, + const std::string& zoneId, int fetchSize = DEFAULT_FETCH_SIZE) { this->host = host; this->rpcPort = rpcPort; this->username = username; @@ -1030,10 +1157,12 @@ class Session { this->fetchSize = fetchSize; this->version = Version::V_1_0; initZoneId(); + initNodesSupplier(); } - Session(const std::string &host, const std::string &rpcPort, const std::string &username = "user", - const std::string &password = "password", const std::string &zoneId="", int fetchSize = DEFAULT_FETCH_SIZE) { + Session(const std::string& host, const std::string& rpcPort, const std::string& username = "user", + const std::string& password = "password", const std::string& zoneId = "", + int fetchSize = DEFAULT_FETCH_SIZE) { this->host = host; this->rpcPort = stoi(rpcPort); this->username = username; @@ -1042,11 +1171,26 @@ class Session { this->fetchSize = fetchSize; this->version = Version::V_1_0; initZoneId(); + initNodesSupplier(); } - ~Session(); + Session(AbstractSessionBuilder* builder) { + this->host = builder->host; + this->rpcPort = builder->rpcPort; + this->username = builder->username; + this->password = builder->password; + this->zoneId = builder->zoneId; + this->fetchSize = builder->fetchSize; + this->version = Version::V_1_0; + this->sqlDialect = builder->sqlDialect; + this->database = builder->database; + this->enableAutoFetch = builder->enableAutoFetch; + this->enableRedirection = builder->enableRedirections; + initZoneId(); + initNodesSupplier(); + } - int64_t getSessionId(); + ~Session(); void open(); @@ -1056,196 +1200,299 @@ class Session { void close(); - void setTimeZone(const std::string &zoneId); + void setTimeZone(const std::string& zoneId); std::string getTimeZone(); - void insertRecord(const std::string &deviceId, int64_t time, const std::vector &measurements, - const std::vector &values); - - void insertRecord(const std::string &deviceId, int64_t time, const std::vector &measurements, - const std::vector &types, const std::vector &values); - - void insertAlignedRecord(const std::string &deviceId, int64_t time, const std::vector &measurements, - const std::vector &values); - - void insertAlignedRecord(const std::string &deviceId, int64_t time, const std::vector &measurements, - const std::vector &types, const std::vector &values); - - void insertRecords(const std::vector &deviceIds, - const std::vector ×, - const std::vector> &measurementsList, - const std::vector> &valuesList); - - void insertRecords(const std::vector &deviceIds, - const std::vector ×, - const std::vector> &measurementsList, - const std::vector> &typesList, - const std::vector> &valuesList); - - void insertAlignedRecords(const std::vector &deviceIds, - const std::vector ×, - const std::vector> &measurementsList, - const std::vector> &valuesList); - - void insertAlignedRecords(const std::vector &deviceIds, - const std::vector ×, - const std::vector> &measurementsList, - const std::vector> &typesList, - const std::vector> &valuesList); - - void insertRecordsOfOneDevice(const std::string &deviceId, - std::vector ×, - std::vector> &measurementsList, - std::vector> &typesList, - std::vector> &valuesList); - - void insertRecordsOfOneDevice(const std::string &deviceId, - std::vector ×, - std::vector> &measurementsList, - std::vector> &typesList, - std::vector> &valuesList, + void insertRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, + const std::vector& values); + + void insertRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, + const std::vector& types, const std::vector& values); + + void insertAlignedRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, + const std::vector& values); + + void insertAlignedRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, + const std::vector& types, const std::vector& values); + + void insertRecords(const std::vector& deviceIds, + const std::vector& times, + const std::vector>& measurementsList, + const std::vector>& valuesList); + + void insertRecords(const std::vector& deviceIds, + const std::vector& times, + const std::vector>& measurementsList, + const std::vector>& typesList, + const std::vector>& valuesList); + + void insertAlignedRecords(const std::vector& deviceIds, + const std::vector& times, + const std::vector>& measurementsList, + const std::vector>& valuesList); + + void insertAlignedRecords(const std::vector& deviceIds, + const std::vector& times, + const std::vector>& measurementsList, + const std::vector>& typesList, + const std::vector>& valuesList); + + void insertRecordsOfOneDevice(const std::string& deviceId, + std::vector& times, + std::vector>& measurementsList, + std::vector>& typesList, + std::vector>& valuesList); + + void insertRecordsOfOneDevice(const std::string& deviceId, + std::vector& times, + std::vector>& measurementsList, + std::vector>& typesList, + std::vector>& valuesList, bool sorted); - void insertAlignedRecordsOfOneDevice(const std::string &deviceId, - std::vector ×, - std::vector> &measurementsList, - std::vector> &typesList, - std::vector> &valuesList); - - void insertAlignedRecordsOfOneDevice(const std::string &deviceId, - std::vector ×, - std::vector> &measurementsList, - std::vector> &typesList, - std::vector> &valuesList, + void insertAlignedRecordsOfOneDevice(const std::string& deviceId, + std::vector& times, + std::vector>& measurementsList, + std::vector>& typesList, + std::vector>& valuesList); + + void insertAlignedRecordsOfOneDevice(const std::string& deviceId, + std::vector& times, + std::vector>& measurementsList, + std::vector>& typesList, + std::vector>& valuesList, bool sorted); - void insertTablet(Tablet &tablet); + void insertTablet(Tablet& tablet); + + void insertTablet(Tablet& tablet, bool sorted); + + void insertRelationalTablet(Tablet& tablet); + + void insertRelationalTabletOnce( + const std::unordered_map, Tablet>& relationalTabletGroup, + bool sorted); + + void insertRelationalTabletByGroup( + const std::unordered_map, Tablet>& relationalTabletGroup, + bool sorted); + + void insertRelationalTablet(Tablet& tablet, bool sorted); + + static void buildInsertTabletReq(TSInsertTabletReq& request, Tablet& tablet, bool sorted); - void insertTablet(Tablet &tablet, bool sorted); + void insertTablet(TSInsertTabletReq request); - static void buildInsertTabletReq(TSInsertTabletReq &request, int64_t sessionId, Tablet &tablet, bool sorted); + void insertAlignedTablet(Tablet& tablet); - void insertTablet(const TSInsertTabletReq &request); + void insertAlignedTablet(Tablet& tablet, bool sorted); - void insertAlignedTablet(Tablet &tablet); + void insertTablets(std::unordered_map& tablets); - void insertAlignedTablet(Tablet &tablet, bool sorted); + void insertTablets(std::unordered_map& tablets, bool sorted); - void insertTablets(std::unordered_map &tablets); + void insertAlignedTablets(std::unordered_map& tablets, bool sorted = false); - void insertTablets(std::unordered_map &tablets, bool sorted); + void testInsertRecord(const std::string& deviceId, int64_t time, + const std::vector& measurements, + const std::vector& values); - void insertAlignedTablets(std::unordered_map &tablets, bool sorted = false); + void testInsertTablet(const Tablet& tablet); - void testInsertRecord(const std::string &deviceId, int64_t time, - const std::vector &measurements, - const std::vector &values); + void testInsertRecords(const std::vector& deviceIds, + const std::vector& times, + const std::vector>& measurementsList, + const std::vector>& valuesList); - void testInsertTablet(const Tablet &tablet); + void deleteTimeseries(const std::string& path); - void testInsertRecords(const std::vector &deviceIds, - const std::vector ×, - const std::vector> &measurementsList, - const std::vector> &valuesList); + void deleteTimeseries(const std::vector& paths); - void deleteTimeseries(const std::string &path); + void deleteData(const std::string& path, int64_t endTime); - void deleteTimeseries(const std::vector &paths); + void deleteData(const std::vector& paths, int64_t endTime); - void deleteData(const std::string &path, int64_t endTime); + void deleteData(const std::vector& paths, int64_t startTime, int64_t endTime); - void deleteData(const std::vector &paths, int64_t endTime); + void setStorageGroup(const std::string& storageGroupId); - void deleteData(const std::vector &paths, int64_t startTime, int64_t endTime); + void deleteStorageGroup(const std::string& storageGroup); - void setStorageGroup(const std::string &storageGroupId); + void deleteStorageGroups(const std::vector& storageGroups); - void deleteStorageGroup(const std::string &storageGroup); + void createDatabase(const std::string& database); - void deleteStorageGroups(const std::vector &storageGroups); + void deleteDatabase(const std::string& database); - void createTimeseries(const std::string &path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, + void deleteDatabases(const std::vector& databases); + + void createTimeseries(const std::string& path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor); - void createTimeseries(const std::string &path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, + void createTimeseries(const std::string& path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor, - std::map *props, std::map *tags, - std::map *attributes, - const std::string &measurementAlias); + std::map* props, std::map* tags, + std::map* attributes, + const std::string& measurementAlias); + + void createMultiTimeseries(const std::vector& paths, + const std::vector& dataTypes, + const std::vector& encodings, + const std::vector& compressors, + std::vector>* propsList, + std::vector>* tagsList, + std::vector>* attributesList, + std::vector* measurementAliasList); - void createMultiTimeseries(const std::vector &paths, - const std::vector &dataTypes, - const std::vector &encodings, - const std::vector &compressors, - std::vector> *propsList, - std::vector> *tagsList, - std::vector> *attributesList, - std::vector *measurementAliasList); + void createAlignedTimeseries(const std::string& deviceId, + const std::vector& measurements, + const std::vector& dataTypes, + const std::vector& encodings, + const std::vector& compressors); - void createAlignedTimeseries(const std::string &deviceId, - const std::vector &measurements, - const std::vector &dataTypes, - const std::vector &encodings, - const std::vector &compressors); + bool checkTimeseriesExists(const std::string& path); - bool checkTimeseriesExists(const std::string &path); + std::unique_ptr executeQueryStatement(const std::string& sql); - std::unique_ptr executeQueryStatement(const std::string &sql) ; + std::unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs); - std::unique_ptr executeQueryStatement(const std::string &sql, int64_t timeoutInMs) ; + std::unique_ptr executeQueryStatementMayRedirect(const std::string& sql, int64_t timeoutInMs); - void executeNonQueryStatement(const std::string &sql); + void executeNonQueryStatement(const std::string& sql); - std::unique_ptr executeRawDataQuery(const std::vector &paths, int64_t startTime, int64_t endTime); + std::unique_ptr executeRawDataQuery(const std::vector& paths, int64_t startTime, + int64_t endTime); - std::unique_ptr executeLastDataQuery(const std::vector &paths); + std::unique_ptr executeLastDataQuery(const std::vector& paths); - std::unique_ptr executeLastDataQuery(const std::vector &paths, int64_t lastTime); + std::unique_ptr executeLastDataQuery(const std::vector& paths, int64_t lastTime); - void createSchemaTemplate(const Template &templ); + void createSchemaTemplate(const Template& templ); - void setSchemaTemplate(const std::string &template_name, const std::string &prefix_path); + void setSchemaTemplate(const std::string& template_name, const std::string& prefix_path); - void unsetSchemaTemplate(const std::string &prefix_path, const std::string &template_name); + void unsetSchemaTemplate(const std::string& prefix_path, const std::string& template_name); - void addAlignedMeasurementsInTemplate(const std::string &template_name, - const std::vector &measurements, - const std::vector &dataTypes, - const std::vector &encodings, - const std::vector &compressors); + void addAlignedMeasurementsInTemplate(const std::string& template_name, + const std::vector& measurements, + const std::vector& dataTypes, + const std::vector& encodings, + const std::vector& compressors); - void addAlignedMeasurementsInTemplate(const std::string &template_name, - const std::string &measurement, + void addAlignedMeasurementsInTemplate(const std::string& template_name, + const std::string& measurement, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor); - void addUnalignedMeasurementsInTemplate(const std::string &template_name, - const std::vector &measurements, - const std::vector &dataTypes, - const std::vector &encodings, - const std::vector &compressors); + void addUnalignedMeasurementsInTemplate(const std::string& template_name, + const std::vector& measurements, + const std::vector& dataTypes, + const std::vector& encodings, + const std::vector& compressors); - void addUnalignedMeasurementsInTemplate(const std::string &template_name, - const std::string &measurement, + void addUnalignedMeasurementsInTemplate(const std::string& template_name, + const std::string& measurement, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, CompressionType::CompressionType compressor); - void deleteNodeInTemplate(const std::string &template_name, const std::string &path); + void deleteNodeInTemplate(const std::string& template_name, const std::string& path); - int countMeasurementsInTemplate(const std::string &template_name); + int countMeasurementsInTemplate(const std::string& template_name); - bool isMeasurementInTemplate(const std::string &template_name, const std::string &path); + bool isMeasurementInTemplate(const std::string& template_name, const std::string& path); - bool isPathExistInTemplate(const std::string &template_name, const std::string &path); + bool isPathExistInTemplate(const std::string& template_name, const std::string& path); - std::vector showMeasurementsInTemplate(const std::string &template_name); + std::vector showMeasurementsInTemplate(const std::string& template_name); - std::vector showMeasurementsInTemplate(const std::string &template_name, const std::string &pattern); + std::vector showMeasurementsInTemplate(const std::string& template_name, const std::string& pattern); - bool checkTemplateExists(const std::string &template_name); + bool checkTemplateExists(const std::string& template_name); }; +template +void Session::insertByGroup(std::unordered_map, T>& insertGroup, + InsertConsumer insertConsumer) { + std::vector> futures; + + for (auto& entry : insertGroup) { + auto connection = entry.first; + auto& req = entry.second; + futures.emplace_back(std::async(std::launch::async, [=, &req]() mutable { + try { + insertConsumer(connection, req); + } + catch (const RedirectException& e) { + for (const auto& deviceEndPoint : e.deviceEndPointMap) { + handleRedirection(deviceEndPoint.first, deviceEndPoint.second); + } + } catch (const IoTDBConnectionException& e) { + if (endPointToSessionConnection.size() > 1) { + removeBrokenSessionConnection(connection); + try { + insertConsumer(defaultSessionConnection, req); + } + catch (const RedirectException&) { + } + } + else { + throw; + } + } catch (const std::exception& e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } + })); + } + + std::string errorMessages; + for (auto& f : futures) { + try { + f.get(); + } + catch (const IoTDBConnectionException& e) { + throw; + } catch (const std::exception& e) { + if (!errorMessages.empty()) { + errorMessages += ";"; + } + errorMessages += e.what(); + } + } + + if (!errorMessages.empty()) { + throw StatementExecutionException(errorMessages); + } +} + +template +void Session::insertOnce(std::unordered_map, T>& insertGroup, + InsertConsumer insertConsumer) { + auto connection = insertGroup.begin()->first; + auto req = insertGroup.begin()->second; + try { + insertConsumer(connection, req); + } + catch (RedirectException e) { + for (const auto& deviceEndPoint : e.deviceEndPointMap) { + handleRedirection(deviceEndPoint.first, deviceEndPoint.second); + } + } catch (IoTDBConnectionException e) { + if (endPointToSessionConnection.size() > 1) { + removeBrokenSessionConnection(connection); + try { + insertConsumer(defaultSessionConnection, req); + } + catch (RedirectException e) { + } + } + else { + throw e; + } + } +} + #endif // IOTDB_SESSION_H diff --git a/iotdb-client/client-cpp/src/main/SessionConnection.cpp b/iotdb-client/client-cpp/src/main/SessionConnection.cpp new file mode 100644 index 0000000000000..4090c4c92f679 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/SessionConnection.cpp @@ -0,0 +1,675 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "SessionConnection.h" +#include "Session.h" +#include "common_types.h" +#include + +#include + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +SessionConnection::SessionConnection(Session* session_ptr, const TEndPoint& endpoint, + const std::string& zoneId, + std::shared_ptr nodeSupplier, + int fetchSize, + int maxRetries, + int64_t retryInterval, + std::string dialect, + std::string db) + : session(session_ptr), + zoneId(zoneId), + endPoint(endpoint), + availableNodes(std::move(nodeSupplier)), + fetchSize(fetchSize), + maxRetryCount(maxRetries), + retryIntervalMs(retryInterval), + sqlDialect(std::move(dialect)), + database(std::move(db)) { + this->zoneId = zoneId.empty() ? getSystemDefaultZoneId() : zoneId; + endPointList.push_back(endpoint); + init(endPoint); +} + +void SessionConnection::close() { + bool needThrowException = false; + string errMsg; + session = nullptr; + try { + TSCloseSessionReq req; + req.__set_sessionId(sessionId); + TSStatus tsStatus; + client->closeSession(tsStatus, req); + } + catch (const TTransportException& e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const exception& e) { + log_debug(e.what()); + errMsg = errMsg + "Session::close() client->closeSession() error, maybe remote server is down. " + e.what() + + "\n"; + needThrowException = true; + } + + try { + if (transport->isOpen()) { + transport->close(); + } + } + catch (const exception& e) { + log_debug(e.what()); + errMsg = errMsg + "Session::close() transport->close() error. " + e.what() + "\n"; + needThrowException = true; + } + + if (needThrowException) { + throw IoTDBException(errMsg); + } +} + +SessionConnection::~SessionConnection() { + try { + close(); + } + catch (const exception& e) { + log_debug(e.what()); + } +} + +void SessionConnection::init(const TEndPoint& endpoint) { + shared_ptr socket(new TSocket(endpoint.ip, endpoint.port)); + transport = std::make_shared(socket); + socket->setConnTimeout(connectionTimeoutInMs); + if (!transport->isOpen()) { + try { + transport->open(); + } + catch (TTransportException& e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } + } + if (enableRPCCompression) { + shared_ptr protocol(new TCompactProtocol(transport)); + client = std::make_shared(protocol); + } + else { + shared_ptr protocol(new TBinaryProtocol(transport)); + client = std::make_shared(protocol); + } + + std::map configuration; + configuration["version"] = session->getVersionString(session->version); + configuration["sql_dialect"] = sqlDialect; + if (database != "") { + configuration["db"] = database; + } + TSOpenSessionReq openReq; + openReq.__set_username(session->username); + openReq.__set_password(session->password); + openReq.__set_zoneId(zoneId); + openReq.__set_configuration(configuration); + try { + TSOpenSessionResp openResp; + client->openSession(openResp, openReq); + RpcUtils::verifySuccess(openResp.status); + if (session->protocolVersion != openResp.serverProtocolVersion) { + if (openResp.serverProtocolVersion == 0) { + // less than 0.10 + throw logic_error(string("Protocol not supported, Client version is ") + + to_string(session->protocolVersion) + + ", but Server version is " + to_string(openResp.serverProtocolVersion)); + } + } + + sessionId = openResp.sessionId; + statementId = client->requestStatementId(sessionId); + + if (!zoneId.empty()) { + setTimeZone(zoneId); + } + } + catch (const TTransportException& e) { + log_debug(e.what()); + transport->close(); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException& e) { + log_debug(e.what()); + transport->close(); + throw; + } catch (const exception& e) { + log_debug(e.what()); + transport->close(); + throw; + } +} + +std::unique_ptr SessionConnection::executeQueryStatement(const std::string& sql, int64_t timeoutInMs) { + TSExecuteStatementReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_statement(sql); + req.__set_timeout(timeoutInMs); + req.__set_enableRedirectQuery(true); + + auto result = callWithRetryAndReconnect( + [this, &req]() { + TSExecuteStatementResp resp; + client->executeStatement(resp, req); + return resp; + }, + [](const TSExecuteStatementResp& resp) { + return resp.status; + } + ); + TSExecuteStatementResp resp = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(resp.status); + } + else { + RpcUtils::verifySuccess(resp.status); + } + + std::shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); + return std::unique_ptr(new SessionDataSet( + sql, resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, resp.queryId, + statementId, client, sessionId, queryDataSet)); +} + +std::unique_ptr SessionConnection::executeRawDataQuery(const std::vector& paths, + int64_t startTime, int64_t endTime) { + TSRawDataQueryReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_fetchSize(fetchSize); + req.__set_paths(paths); + req.__set_startTime(startTime); + req.__set_endTime(endTime); + auto result = callWithRetryAndReconnect( + [this, &req]() { + TSExecuteStatementResp resp; + client->executeRawDataQuery(resp, req); + return resp; + }, + [](const TSExecuteStatementResp& resp) { + return resp.status; + } + ); + TSExecuteStatementResp resp = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(resp.status); + } + else { + RpcUtils::verifySuccess(resp.status); + } + shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); + return unique_ptr( + new SessionDataSet("", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, + resp.queryId, statementId, client, sessionId, queryDataSet)); +} + +std::unique_ptr SessionConnection::executeLastDataQuery(const std::vector& paths, + int64_t lastTime) { + TSLastDataQueryReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_fetchSize(fetchSize); + req.__set_paths(paths); + req.__set_time(lastTime); + + auto result = callWithRetryAndReconnect( + [this, &req]() { + TSExecuteStatementResp resp; + client->executeLastDataQuery(resp, req); + return resp; + }, + [](const TSExecuteStatementResp& resp) { + return resp.status; + } + ); + TSExecuteStatementResp resp = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(resp.status); + } + else { + RpcUtils::verifySuccess(resp.status); + } + shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); + return unique_ptr( + new SessionDataSet("", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, + resp.queryId, statementId, client, sessionId, queryDataSet)); +} + +void SessionConnection::executeNonQueryStatement(const string& sql) { + TSExecuteStatementReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_statement(sql); + req.__set_timeout(0); //0 means no timeout. This value keep consistent to JAVA SDK. + TSExecuteStatementResp resp; + try { + client->executeUpdateStatementV2(resp, req); + if (resp.database != "") { + database = resp.database; + session->database = database; + } + RpcUtils::verifySuccess(resp.status); + } + catch (const TTransportException& e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException& e) { + log_debug(e.what()); + throw; + } catch (const exception& e) { + throw IoTDBException(e.what()); + } +} + +const TEndPoint& SessionConnection::getEndPoint() { + return endPoint; +} + +void SessionConnection::setTimeZone(const std::string& newZoneId) { + TSSetTimeZoneReq req; + req.__set_sessionId(sessionId); + req.__set_timeZone(newZoneId); + + try { + TSStatus tsStatus; + client->setTimeZone(tsStatus, req); + zoneId = newZoneId; + } + catch (const TException& e) { + throw IoTDBConnectionException(e.what()); + } +} + +std::string SessionConnection::getSystemDefaultZoneId() { + time_t ts = 0; + struct tm tmv{}; +#if defined(_WIN64) || defined (WIN32) || defined (_WIN32) + localtime_s(&tmv, &ts); +#else + localtime_r(&ts, &tmv); +#endif + char zoneStr[32]; + strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); + return zoneStr; +} + +bool SessionConnection::reconnect() { + bool reconnect = false; + for (int i = 1; i <= 3; i++) { + if (transport != nullptr) { + transport->close(); + endPointList = std::move(availableNodes->getEndPointList()); + int currHostIndex = rand() % endPointList.size(); + int tryHostNum = 0; + for (int j = currHostIndex; j < endPointList.size(); j++) { + if (tryHostNum == endPointList.size()) { + break; + } + this->endPoint = endPointList[j]; + if (j == endPointList.size() - 1) { + j = -1; + } + tryHostNum++; + try { + init(this->endPoint); + reconnect = true; + } + catch (const IoTDBConnectionException& e) { + log_warn("The current node may have been down, connection exception: %s", e.what()); + continue; + } catch (exception& e) { + log_warn("login in failed, because %s", e.what()); + } + break; + } + } + if (reconnect) { + session->removeBrokenSessionConnection(shared_from_this()); + session->defaultEndPoint = this->endPoint; + session->defaultSessionConnection = shared_from_this(); + session->endPointToSessionConnection.insert(make_pair(this->endPoint, shared_from_this())); + } + } + return reconnect; +} + +void SessionConnection::insertStringRecord(const TSInsertStringRecordReq& request) { + auto rpc = [this, request]() { + return this->insertStringRecordInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertRecord(const TSInsertRecordReq& request) { + auto rpc = [this, request]() { + return this->insertRecordInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertStringRecords(const TSInsertStringRecordsReq& request) { + auto rpc = [this, request]() { + return this->insertStringRecordsInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertRecords(const TSInsertRecordsReq& request) { + auto rpc = [this, request]() { + return this->insertRecordsInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq request) { + auto rpc = [this, request]() { + return this->insertRecordsOfOneDeviceInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceReq request) { + auto rpc = [this, request]() { + return this->insertStringRecordsOfOneDeviceInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertTablet(TSInsertTabletReq request) { + auto rpc = [this, request]() { + return this->insertTabletInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertTablets(TSInsertTabletsReq request) { + auto rpc = [this, request]() { + return this->insertTabletsInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::testInsertStringRecord(TSInsertStringRecordReq& request) { + auto rpc = [this, &request]() { + request.sessionId = sessionId; + TSStatus ret; + client->testInsertStringRecord(ret, request); + return ret; + }; + auto status = callWithRetryAndReconnect(rpc).getResult(); + RpcUtils::verifySuccess(status); +} + +void SessionConnection::testInsertTablet(TSInsertTabletReq& request) { + auto rpc = [this, &request]() { + request.sessionId = sessionId; + TSStatus ret; + client->testInsertTablet(ret, request); + return ret; + }; + auto status = callWithRetryAndReconnect(rpc).getResult(); + RpcUtils::verifySuccess(status); +} + +void SessionConnection::testInsertRecords(TSInsertRecordsReq& request) { + auto rpc = [this, &request]() { + request.sessionId = sessionId; + TSStatus ret; + client->testInsertRecords(ret, request); + return ret; + }; + auto status = callWithRetryAndReconnect(rpc).getResult(); + RpcUtils::verifySuccess(status); +} + +void SessionConnection::deleteTimeseries(const vector& paths) { + auto rpc = [this, &paths]() { + TSStatus ret; + client->deleteTimeseries(ret, sessionId, paths); + return ret; + }; + callWithRetryAndVerify(rpc); +} + +void SessionConnection::deleteData(const TSDeleteDataReq& request) { + auto rpc = [this, request]() { + return this->deleteDataInternal(request); + }; + callWithRetryAndVerify(rpc); +} + +void SessionConnection::setStorageGroup(const string& storageGroupId) { + auto rpc = [this, &storageGroupId]() { + TSStatus ret; + client->setStorageGroup(ret, sessionId, storageGroupId); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::deleteStorageGroups(const vector& storageGroups) { + auto rpc = [this, &storageGroups]() { + TSStatus ret; + client->deleteStorageGroups(ret, sessionId, storageGroups); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createTimeseries(TSCreateTimeseriesReq& req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createTimeseries(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createMultiTimeseries(TSCreateMultiTimeseriesReq& req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createMultiTimeseries(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createAlignedTimeseries(TSCreateAlignedTimeseriesReq& req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createAlignedTimeseries(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +TSGetTimeZoneResp SessionConnection::getTimeZone() { + auto rpc = [this]() { + TSGetTimeZoneResp resp; + client->getTimeZone(resp, sessionId); + zoneId = resp.timeZone; + return resp; + }; + auto ret = callWithRetryAndReconnect(rpc, + [](const TSGetTimeZoneResp& resp) { + return resp.status; + }); + RpcUtils::verifySuccess(ret.getResult().status); + return ret.result; +} + +void SessionConnection::setTimeZone(TSSetTimeZoneReq& req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->setTimeZone(ret, req); + zoneId = req.timeZone; + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createSchemaTemplate(TSCreateSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::setSchemaTemplate(TSSetSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->setSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::unsetSchemaTemplate(TSUnsetSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->unsetSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::appendSchemaTemplate(TSAppendSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->appendSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::pruneSchemaTemplate(TSPruneSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->pruneSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +TSQueryTemplateResp SessionConnection::querySchemaTemplate(TSQueryTemplateReq req) { + auto rpc = [this, &req]() { + TSQueryTemplateResp ret; + req.sessionId = sessionId; + client->querySchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc, + [](const TSQueryTemplateResp& resp) { + return resp.status; + }); + RpcUtils::verifySuccess(ret.getResult().status); + return ret.getResult(); +} + +TSStatus SessionConnection::insertStringRecordInternal(TSInsertStringRecordReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertStringRecord(ret, request); + return ret; +} + +TSStatus SessionConnection::insertRecordInternal(TSInsertRecordReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertRecord(ret, request); + return ret; +} + +TSStatus SessionConnection::insertStringRecordsInternal(TSInsertStringRecordsReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertStringRecords(ret, request); + return ret; +} + +TSStatus SessionConnection::insertRecordsInternal(TSInsertRecordsReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertRecords(ret, request); + return ret; +} + +TSStatus SessionConnection::insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertRecordsOfOneDevice(ret, request); + return ret; +} + +TSStatus SessionConnection::insertStringRecordsOfOneDeviceInternal(TSInsertStringRecordsOfOneDeviceReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertStringRecordsOfOneDevice(ret, request); + return ret; +} + +TSStatus SessionConnection::insertTabletInternal(TSInsertTabletReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertTablet(ret, request); + return ret; +} + +TSStatus SessionConnection::insertTabletsInternal(TSInsertTabletsReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertTablets(ret, request); + return ret; +} + +TSStatus SessionConnection::deleteDataInternal(TSDeleteDataReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->deleteData(ret, request); + return ret; +} diff --git a/iotdb-client/client-cpp/src/main/SessionConnection.h b/iotdb-client/client-cpp/src/main/SessionConnection.h new file mode 100644 index 0000000000000..6bb9ccd980533 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/SessionConnection.h @@ -0,0 +1,363 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_SESSIONCONNECTION_H +#define IOTDB_SESSIONCONNECTION_H + +#include +#include +#include +#include +#include "IClientRPCService.h" +#include "common_types.h" +#include "NodesSupplier.h" +#include "Common.h" + +class SessionDataSet; +class Session; + +class SessionConnection : public std::enable_shared_from_this { +public: + SessionConnection(Session* session_ptr, const TEndPoint& endpoint, + const std::string& zoneId, + std::shared_ptr nodeSupplier, + int fetchSize = 10000, + int maxRetries = 60, + int64_t retryInterval = 500, + std::string dialect = "tree", + std::string db = ""); + + ~SessionConnection(); + + void setTimeZone(const std::string& newZoneId); + + + const TEndPoint& getEndPoint(); + + void init(const TEndPoint& endpoint); + + void insertStringRecord(const TSInsertStringRecordReq& request); + + void insertRecord(const TSInsertRecordReq& request); + + void insertStringRecords(const TSInsertStringRecordsReq& request); + + void insertRecords(const TSInsertRecordsReq& request); + + void insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq request); + + void insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceReq request); + + void insertTablet(TSInsertTabletReq request); + + void insertTablets(TSInsertTabletsReq request); + + void testInsertStringRecord(TSInsertStringRecordReq& request); + + void testInsertTablet(TSInsertTabletReq& request); + + void testInsertRecords(TSInsertRecordsReq& request); + + void deleteTimeseries(const vector& paths); + + void deleteData(const TSDeleteDataReq& request); + + void setStorageGroup(const string& storageGroupId); + + void deleteStorageGroups(const vector& storageGroups); + + void createTimeseries(TSCreateTimeseriesReq& req); + + void createMultiTimeseries(TSCreateMultiTimeseriesReq& req); + + void createAlignedTimeseries(TSCreateAlignedTimeseriesReq& req); + + TSGetTimeZoneResp getTimeZone(); + + void setTimeZone(TSSetTimeZoneReq& req); + + void createSchemaTemplate(TSCreateSchemaTemplateReq req); + + void setSchemaTemplate(TSSetSchemaTemplateReq req); + + void unsetSchemaTemplate(TSUnsetSchemaTemplateReq req); + + void appendSchemaTemplate(TSAppendSchemaTemplateReq req); + + void pruneSchemaTemplate(TSPruneSchemaTemplateReq req); + + TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req); + + std::unique_ptr executeRawDataQuery(const std::vector& paths, int64_t startTime, + int64_t endTime); + + std::unique_ptr executeLastDataQuery(const std::vector& paths, int64_t lastTime); + + void executeNonQueryStatement(const std::string& sql); + + std::unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs = -1); + + std::shared_ptr getSessionClient() { + return client; + } + + friend class Session; + +private: + void close(); + std::string getSystemDefaultZoneId(); + bool reconnect(); + + template + struct RetryResult { + T result; + std::exception_ptr exception; + int retryAttempts; + + RetryResult(T r, std::exception_ptr e, int a) + : result(r), exception(e), retryAttempts(a) { + } + + int getRetryAttempts() const { return retryAttempts; } + T getResult() const { return result; } + std::exception_ptr getException() const { return exception; } + }; + + template + void callWithRetryAndVerifyWithRedirection(std::function rpc); + + template + void callWithRetryAndVerifyWithRedirectionForMultipleDevices( + std::function rpc, const vector& deviceIds); + + template + RetryResult callWithRetryAndVerify(std::function rpc); + + template + RetryResult callWithRetry(std::function rpc); + + template + RetryResult callWithRetryAndReconnect(RpcFunc rpc); + + template + RetryResult callWithRetryAndReconnect(RpcFunc rpc, StatusGetter statusGetter); + + template + RetryResult callWithRetryAndReconnect(RpcFunc rpc, ShouldRetry shouldRetry, ForceReconnect forceReconnect); + + TSStatus insertStringRecordInternal(TSInsertStringRecordReq request); + + TSStatus insertRecordInternal(TSInsertRecordReq request); + + TSStatus insertStringRecordsInternal(TSInsertStringRecordsReq request); + + TSStatus insertRecordsInternal(TSInsertRecordsReq request); + + TSStatus insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq request); + + TSStatus insertStringRecordsOfOneDeviceInternal(TSInsertStringRecordsOfOneDeviceReq request); + + TSStatus insertTabletInternal(TSInsertTabletReq request); + + TSStatus insertTabletsInternal(TSInsertTabletsReq request); + + TSStatus deleteDataInternal(TSDeleteDataReq request); + + std::shared_ptr transport; + std::shared_ptr client; + Session* session; + int64_t sessionId; + int64_t statementId; + int64_t connectionTimeoutInMs; + bool enableRPCCompression = false; + std::string zoneId; + TEndPoint endPoint; + std::vector endPointList; + std::shared_ptr availableNodes; + int fetchSize; + int maxRetryCount; + int64_t retryIntervalMs; + std::string sqlDialect; + std::string database; +}; + +template +SessionConnection::RetryResult SessionConnection::callWithRetry(std::function rpc) { + std::exception_ptr lastException = nullptr; + TSStatus status; + int i; + for (i = 0; i <= maxRetryCount; i++) { + if (i > 0) { + lastException = nullptr; + status = TSStatus(); + try { + std::this_thread::sleep_for( + std::chrono::milliseconds(retryIntervalMs)); + } + catch (const std::exception& e) { + break; + } + if (!reconnect()) { + continue; + } + } + + try { + status = rpc(); + if (status.__isset.needRetry && status.needRetry) { + continue; + } + break; + } + catch (...) { + lastException = std::current_exception(); + } + } + return {status, lastException, i}; +} + +template +void SessionConnection::callWithRetryAndVerifyWithRedirection(std::function rpc) { + auto result = callWithRetry(rpc); + + auto status = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(status); + } + else { + RpcUtils::verifySuccess(status); + } + + if (result.getException()) { + try { + std::rethrow_exception(result.getException()); + } + catch (const std::exception& e) { + throw IoTDBConnectionException(e.what()); + } + } +} + +template +void SessionConnection::callWithRetryAndVerifyWithRedirectionForMultipleDevices( + std::function rpc, const vector& deviceIds) { + auto result = callWithRetry(rpc); + auto status = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirectionForMultiDevices(status, deviceIds); + } + else { + RpcUtils::verifySuccess(status); + } + if (result.getException()) { + try { + std::rethrow_exception(result.getException()); + } + catch (const std::exception& e) { + throw IoTDBConnectionException(e.what()); + } + } + result.exception = nullptr; +} + +template +SessionConnection::RetryResult SessionConnection::callWithRetryAndVerify(std::function rpc) { + auto result = callWithRetry(rpc); + RpcUtils::verifySuccess(result.getResult()); + if (result.getException()) { + try { + std::rethrow_exception(result.getException()); + } + catch (const std::exception& e) { + throw IoTDBConnectionException(e.what()); + } + } + return result; +} + +template +SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect(RpcFunc rpc) { + return callWithRetryAndReconnect(rpc, + [](const TSStatus& status) { + return status.__isset.needRetry && status.needRetry; + }, + [](const TSStatus& status) { + return status.code == TSStatusCode::PLAN_FAILED_NETWORK_PARTITION; + } + ); +} + +template +SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect(RpcFunc rpc, StatusGetter statusGetter) { + auto shouldRetry = [&statusGetter](const T& t) { + auto status = statusGetter(t); + return status.__isset.needRetry && status.needRetry; + }; + auto forceReconnect = [&statusGetter](const T& t) { + auto status = statusGetter(t); + return status.code == TSStatusCode::PLAN_FAILED_NETWORK_PARTITION;; + }; + return callWithRetryAndReconnect(rpc, shouldRetry, forceReconnect); +} + + +template +SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect(RpcFunc rpc, + ShouldRetry shouldRetry, + ForceReconnect forceReconnect) { + std::exception_ptr lastException = nullptr; + T result; + int retryAttempt; + for (retryAttempt = 0; retryAttempt <= maxRetryCount; retryAttempt++) { + try { + result = rpc(); + lastException = nullptr; + } + catch (...) { + result = T(); + lastException = std::current_exception(); + } + + if (!shouldRetry(result)) { + return {result, lastException, retryAttempt}; + } + + if (lastException != nullptr || + std::find(availableNodes->getEndPointList().begin(), availableNodes->getEndPointList().end(), + this->endPoint) == availableNodes->getEndPointList().end() || + forceReconnect(result)) { + reconnect(); + } + + try { + std::this_thread::sleep_for(std::chrono::milliseconds(retryIntervalMs)); + } + catch (const std::exception& e) { + log_debug("Thread was interrupted during retry " + + std::to_string(retryAttempt) + + " with wait time " + + std::to_string(retryIntervalMs) + + " ms. Exiting retry loop."); + break; + } + } + + return {result, lastException, retryAttempt}; +} + +#endif diff --git a/iotdb-client/client-cpp/src/main/TableSession.cpp b/iotdb-client/client-cpp/src/main/TableSession.cpp new file mode 100644 index 0000000000000..304a2f062adc6 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/TableSession.cpp @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// This file is a translation of the Java file iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java + +#include "TableSession.h" + +void TableSession::insert(Tablet &tablet, bool sorted) { + session->insertRelationalTablet(tablet, sorted); +} +void TableSession::executeNonQueryStatement(const string &sql) { + session->executeNonQueryStatement(sql); +} +unique_ptr TableSession::executeQueryStatement(const string &sql) { + return session->executeQueryStatement(sql); +} +unique_ptr TableSession::executeQueryStatement(const string &sql, int64_t timeoutInMs) { + return session->executeQueryStatement(sql, timeoutInMs); +} +string TableSession::getDatabase() { + return session->getDatabase(); +} +void TableSession::open(bool enableRPCCompression) { + session->open(enableRPCCompression); +} +void TableSession::close() { + session->close(); +} \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/TableSession.h b/iotdb-client/client-cpp/src/main/TableSession.h new file mode 100644 index 0000000000000..abac784feaf4d --- /dev/null +++ b/iotdb-client/client-cpp/src/main/TableSession.h @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// This file is a translation of the Java file iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java + +#ifndef IOTDB_TABLESESSION_H +#define IOTDB_TABLESESSION_H + +#include "Session.h" + +class TableSession { +private: + Session* session; + string getDatabase(); +public: + TableSession(Session* session) { + this->session = session; + } + ~TableSession() { + if (session) { + delete session; + session = nullptr; + } + } + void insert(Tablet& tablet, bool sorted = false); + void executeNonQueryStatement(const std::string& sql); + unique_ptr executeQueryStatement(const std::string& sql); + unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs); + void open(bool enableRPCCompression = false); + void close(); +}; + +#endif // IOTDB_TABLESESSION_H \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/TableSessionBuilder.h b/iotdb-client/client-cpp/src/main/TableSessionBuilder.h new file mode 100644 index 0000000000000..5c1b1a0a22e9f --- /dev/null +++ b/iotdb-client/client-cpp/src/main/TableSessionBuilder.h @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// This file is a translation of the Java file iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java + +#ifndef IOTDB_TABLESESSIONBUILDER_H +#define IOTDB_TABLESESSIONBUILDER_H + +#include "TableSession.h" +#include "AbstractSessionBuilder.h" + +class TableSessionBuilder : public AbstractSessionBuilder { + /* + std::string host; + int rpcPort; + std::string username; + std::string password; + std::string zoneId; + int fetchSize; + std::string sqlDialect = "tree"; // default sql dialect + std::string database; + */ +public: + TableSessionBuilder* host(const std::string &host) { + AbstractSessionBuilder::host = host; + return this; + } + TableSessionBuilder* rpcPort(int rpcPort) { + AbstractSessionBuilder::rpcPort = rpcPort; + return this; + } + TableSessionBuilder* username(const std::string &username) { + AbstractSessionBuilder::username = username; + return this; + } + TableSessionBuilder* password(const std::string &password) { + AbstractSessionBuilder::password = password; + return this; + } + TableSessionBuilder* zoneId(const std::string &zoneId) { + AbstractSessionBuilder::zoneId = zoneId; + return this; + } + TableSessionBuilder* fetchSize(int fetchSize) { + AbstractSessionBuilder::fetchSize = fetchSize; + return this; + } + TableSessionBuilder* database(const std::string &database) { + AbstractSessionBuilder::database = database; + return this; + } + TableSession* build() { + sqlDialect = "table"; + Session* newSession = new Session(this); + newSession->open(false); + return new TableSession(newSession); + } +}; + +#endif // IOTDB_TABLESESSIONBUILDER_H \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/ThriftConnection.cpp b/iotdb-client/client-cpp/src/main/ThriftConnection.cpp new file mode 100644 index 0000000000000..23c2890c34a49 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/ThriftConnection.cpp @@ -0,0 +1,158 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "ThriftConnection.h" +#include +#include +#include + +#include "Session.h" + +const int ThriftConnection::THRIFT_DEFAULT_BUFFER_SIZE = 4096; +const int ThriftConnection::THRIFT_MAX_FRAME_SIZE = 1048576; +const int ThriftConnection::CONNECTION_TIMEOUT_IN_MS = 1000; + +ThriftConnection::ThriftConnection(const TEndPoint& endPoint, + int thriftDefaultBufferSize, + int thriftMaxFrameSize, + int connectionTimeoutInMs) + : endPoint(endPoint), + thriftDefaultBufferSize(thriftDefaultBufferSize), + thriftMaxFrameSize(thriftMaxFrameSize), + connectionTimeoutInMs(connectionTimeoutInMs) {} + +ThriftConnection::~ThriftConnection() = default; + +void ThriftConnection::initZoneId() { + if (!zoneId.empty()) { + return; + } + + time_t ts = 0; + struct tm tmv{}; +#if defined(_WIN64) || defined (WIN32) || defined (_WIN32) + localtime_s(&tmv, &ts); +#else + localtime_r(&ts, &tmv); +#endif + + char zoneStr[32]; + strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); + zoneId = zoneStr; +} + +void ThriftConnection::init(const std::string& username, + const std::string& password, + bool enableRPCCompression, + const std::string& zoneId, + const std::string& version) { + std::shared_ptr socket(new TSocket(endPoint.ip, endPoint.port)); + socket->setConnTimeout(connectionTimeoutInMs); + transport = std::make_shared(socket); + if (!transport->isOpen()) { + try { + transport->open(); + } + catch (TTransportException &e) { + throw IoTDBConnectionException(e.what()); + } + } + if (zoneId.empty()) { + initZoneId(); + } else { + this->zoneId = zoneId; + } + + if (enableRPCCompression) { + std::shared_ptr protocol(new TCompactProtocol(transport)); + client = std::make_shared(protocol); + } else { + std::shared_ptr protocol(new TBinaryProtocol(transport)); + client = std::make_shared(protocol); + } + + std::map configuration; + configuration["version"] = version; + TSOpenSessionReq openReq; + openReq.__set_username(username); + openReq.__set_password(password); + openReq.__set_zoneId(this->zoneId); + openReq.__set_configuration(configuration); + try { + TSOpenSessionResp openResp; + client->openSession(openResp, openReq); + RpcUtils::verifySuccess(openResp.status); + sessionId = openResp.sessionId; + statementId = client->requestStatementId(sessionId); + } catch (const TTransportException &e) { + transport->close(); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + transport->close(); + throw IoTDBException(e.what()); + } catch (const std::exception &e) { + transport->close(); + throw IoTDBException(e.what()); + } +} + +std::unique_ptr ThriftConnection::executeQueryStatement(const std::string& sql, int64_t timeoutInMs) { + TSExecuteStatementReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_statement(sql); + req.__set_timeout(timeoutInMs); + TSExecuteStatementResp resp; + try { + client->executeStatement(resp, req); + RpcUtils::verifySuccess(resp.status); + } catch (const TTransportException &e) { + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + throw IoTDBConnectionException(e.what()); + } catch (const std::exception &e) { + throw IoTDBException(e.what()); + } + std::shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); + return std::unique_ptr(new SessionDataSet( + sql, resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.ignoreTimeStamp, resp.queryId, + statementId, client, sessionId, queryDataSet)); +} + +void ThriftConnection::close() { + try { + if (client) { + TSCloseSessionReq req; + req.__set_sessionId(sessionId); + TSStatus tsStatus; + client->closeSession(tsStatus, req); + } + } catch (const TTransportException &e) { + throw IoTDBConnectionException(e.what()); + } catch (const std::exception &e) { + throw IoTDBConnectionException(e.what()); + } + + try { + if (transport->isOpen()) { + transport->close(); + } + } catch (const std::exception &e) { + throw IoTDBConnectionException(e.what()); + } +} \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/ThriftConnection.h b/iotdb-client/client-cpp/src/main/ThriftConnection.h new file mode 100644 index 0000000000000..2810036078cb8 --- /dev/null +++ b/iotdb-client/client-cpp/src/main/ThriftConnection.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_THRIFTCONNECTION_H +#define IOTDB_THRIFTCONNECTION_H + +#include +#include +#include "IClientRPCService.h" + +class SessionDataSet; + +class ThriftConnection { +public: + static const int THRIFT_DEFAULT_BUFFER_SIZE; + static const int THRIFT_MAX_FRAME_SIZE; + static const int CONNECTION_TIMEOUT_IN_MS; + + explicit ThriftConnection(const TEndPoint& endPoint, + int thriftDefaultBufferSize = THRIFT_DEFAULT_BUFFER_SIZE, + int thriftMaxFrameSize = THRIFT_MAX_FRAME_SIZE, + int connectionTimeoutInMs = CONNECTION_TIMEOUT_IN_MS); + + ~ThriftConnection(); + + void init(const std::string& username, + const std::string& password, + bool enableRPCCompression = false, + const std::string& zoneId = std::string(), + const std::string& version = "V_1_0"); + + std::unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs = -1); + + void close(); + +private: + TEndPoint endPoint; + + int thriftDefaultBufferSize; + int thriftMaxFrameSize; + int connectionTimeoutInMs; + + std::shared_ptr transport; + std::shared_ptr client; + int64_t sessionId{}; + int64_t statementId{}; + std::string zoneId; + int timeFactor{}; + + void initZoneId(); +}; + +#endif diff --git a/iotdb-client/client-cpp/src/test/CMakeLists.txt b/iotdb-client/client-cpp/src/test/CMakeLists.txt index d38c975de201c..186c08b634d46 100644 --- a/iotdb-client/client-cpp/src/test/CMakeLists.txt +++ b/iotdb-client/client-cpp/src/test/CMakeLists.txt @@ -21,6 +21,7 @@ INCLUDE( CTest ) SET(CMAKE_CXX_STANDARD 11) SET(CMAKE_CXX_STANDARD_REQUIRED ON) SET(TARGET_NAME session_tests) +SET(TARGET_NAME_RELATIONAL session_relational_tests) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -g -O2") ENABLE_TESTING() @@ -47,18 +48,26 @@ ELSE() ENDIF() ADD_EXECUTABLE(${TARGET_NAME} main.cpp cpp/sessionIT.cpp) +ADD_EXECUTABLE(${TARGET_NAME_RELATIONAL} main_Relational.cpp cpp/sessionRelationalIT.cpp) # Link with shared library iotdb_session and pthread IF(MSVC) TARGET_LINK_LIBRARIES(${TARGET_NAME} iotdb_session ${THRIFT_STATIC_LIB}) + TARGET_LINK_LIBRARIES(${TARGET_NAME_RELATIONAL} iotdb_session ${THRIFT_STATIC_LIB}) ELSE() TARGET_LINK_LIBRARIES(${TARGET_NAME} iotdb_session pthread) + TARGET_LINK_LIBRARIES(${TARGET_NAME_RELATIONAL} iotdb_session pthread) ENDIF() + +# Add Catch2 include directory TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME} PUBLIC ./catch2/) +TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME_RELATIONAL} PUBLIC ./catch2/) # Add 'sessionIT' to the project to be run by ctest IF(MSVC) ADD_TEST(NAME sessionIT CONFIGURATIONS Release COMMAND ${TARGET_NAME}) + ADD_TEST(NAME sessionRelationalIT CONFIGURATIONS Release COMMAND ${TARGET_NAME_RELATIONAL}) ELSE() ADD_TEST(NAME sessionIT COMMAND ${TARGET_NAME}) + ADD_TEST(NAME sessionRelationalIT COMMAND ${TARGET_NAME_RELATIONAL}) ENDIF() diff --git a/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp b/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp index 8016a4e37b320..83c2443162f8a 100644 --- a/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp +++ b/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp @@ -22,7 +22,7 @@ using namespace std; -extern Session *session; +extern std::shared_ptr session; static vector testTimeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; @@ -176,6 +176,50 @@ TEST_CASE("Test insertRecord with types ", "[testTypedInsertRecord]") { REQUIRE(count == 100); } + +TEST_CASE("Test insertRecord with new datatypes ", "[testTypedInsertRecordNewDatatype]") { + CaseReporter cr("testTypedInsertRecordNewDatatype"); + vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3", "root.test.d1.s4"}; + std::vector types = {TSDataType::TIMESTAMP, + TSDataType::DATE, TSDataType::BLOB, TSDataType::STRING}; + + for (size_t i = 0; i < timeseries.size(); i++) { + if (session->checkTimeseriesExists(timeseries[i])) { + session->deleteTimeseries(timeseries[i]); + } + session->createTimeseries(timeseries[i], types[i], TSEncoding::PLAIN, CompressionType::SNAPPY); + } + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3", "s4"}; + int64_t value1 = 20250507; + boost::gregorian::date value2 = boost::gregorian::date(2025, 5, 7); + string value3 = "20250507"; + string value4 = "20250507"; + + for (long time = 0; time < 100; time++) { + vector values = {(char *) (&value1), (char *) (&value2), + const_cast(value3.c_str()), const_cast(value4.c_str())}; + session->insertRecord(deviceId, time, measurements, types, values); + } + + unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3,s4 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + long count = 0; + while (sessionDataSet->hasNext()) { + auto record = sessionDataSet->next(); + REQUIRE(record->fields.size() == 4); + for (int i = 0; i < 4; i++) { + REQUIRE(types[i] == record->fields[i].dataType); + } + REQUIRE(record->fields[0].longV == value1); + REQUIRE(record->fields[1].dateV == value2); + REQUIRE(record->fields[2].stringV == value3); + REQUIRE(record->fields[3].stringV == value4); + count++; + } + REQUIRE(count == 100); +} + TEST_CASE("Test insertRecords with types ", "[testTypedInsertRecords]") { CaseReporter cr("testTypedInsertRecords"); vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; @@ -274,7 +318,7 @@ TEST_CASE("Test insertTablet ", "[testInsertTablet]") { int row = tablet.rowSize++; tablet.timestamps[row] = time; for (int64_t i = 0; i < 3; i++) { - tablet.addValue(i, row, &i); + tablet.addValue(i, row, i); } if (tablet.rowSize == tablet.maxRowNumber) { session->insertTablet(tablet); @@ -300,6 +344,116 @@ TEST_CASE("Test insertTablet ", "[testInsertTablet]") { REQUIRE(count == 100); } +TEST_CASE("Test insertTablets ", "[testInsertTablets]") { + CaseReporter cr("testInsertTablets"); + vector testTimeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3", + "root.test.d2.s1", "root.test.d2.s2", "root.test.d2.s3"}; + for (const string ×eries: testTimeseries) { + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); + } + session->createTimeseries(timeseries, TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); + } + vector> schemaList; + schemaList.emplace_back("s1", TSDataType::INT64); + schemaList.emplace_back("s2", TSDataType::INT64); + schemaList.emplace_back("s3", TSDataType::INT64); + + int maxRowNumber = 100; + vector deviceIds = {"root.test.d1", "root.test.d2"}; + vector tablets; + for (const auto& deviceId: deviceIds) { + tablets.emplace_back(deviceId, schemaList, maxRowNumber); + } + for (auto& tablet : tablets) { + for (int64_t time = 0; time < maxRowNumber; time++) { + int row = tablet.rowSize++; + tablet.timestamps[row] = time; + for (int64_t i = 0; i < 3; i++) { + tablet.addValue(i, row, i); + } + } + } + unordered_map tabletsMap; + for (auto& tablet : tablets) { + tabletsMap[tablet.deviceId] = &tablet; + } + session->insertTablets(tabletsMap); + + unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d2"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + long index = 0; + count++; + for (const Field& f: sessionDataSet->next()->fields) { + REQUIRE(f.longV == index); + index++; + } + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertTablet new datatype", "[testInsertTabletNewDatatype]") { + CaseReporter cr("testInsertTabletNewDatatype"); + string deviceId = "root.test.d2"; + vector> schemaList; + std::vector measurements = {"s1", "s2", "s3", "s4"}; + std::vector dataTypes = {TSDataType::TIMESTAMP, + TSDataType::DATE, TSDataType::BLOB, TSDataType::STRING}; + for (int i = 0; i < 4; i++) { + schemaList.emplace_back(measurements[i], dataTypes[i]); + } + + for (int i = 0; i < 4; i++) { + auto timeseries = deviceId + "." + measurements[i]; + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); + } + session->createTimeseries(timeseries, dataTypes[i], TSEncoding::PLAIN, CompressionType::UNCOMPRESSED); + } + + int64_t s1Value = 20250507; + boost::gregorian::date s2Value(2025, 5, 7); + std::string s3Value("20250507"); + std::string s4Value("20250507"); + + Tablet tablet(deviceId, schemaList, 100); + for (int64_t time = 0; time < 100; time++) { + int row = tablet.rowSize++; + tablet.timestamps[row] = time; + tablet.addValue(0, row, s1Value); + tablet.addValue(1, row, s2Value); + tablet.addValue(2, row, s3Value); + tablet.addValue(3, row, s4Value); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } + unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3,s4 from root.test.d2"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + auto record = sessionDataSet->next(); + REQUIRE(record->fields.size() == 4); + for (int i = 0; i < 4; i++) { + REQUIRE(dataTypes[i] == record->fields[i].dataType); + } + REQUIRE(record->fields[0].longV == s1Value); + REQUIRE(record->fields[1].dateV == s2Value); + REQUIRE(record->fields[2].stringV == s3Value); + REQUIRE(record->fields[3].stringV == s4Value); + count++; + } + REQUIRE(count == 100); +} + TEST_CASE("Test Last query ", "[testLastQuery]") { CaseReporter cr("testLastQuery"); prepareTimeseries(); diff --git a/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp b/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp new file mode 100644 index 0000000000000..3d5122b6e2671 --- /dev/null +++ b/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "catch.hpp" +#include "TableSession.h" +#include + +using namespace std; + +extern std::shared_ptr session; + +static int global_test_tag = 0; +class CaseReporter +{ +public: + CaseReporter(const char *caseNameArg) : caseName(caseNameArg) + { + test_tag = global_test_tag++; + std::cout << "Test " << test_tag << ": " << caseName << std::endl; + } + ~CaseReporter() + { + std::cout << "Test " << test_tag << ": " << caseName << " Done"<< std::endl << std::endl; + } +private: + const char *caseName; + int test_tag; +}; + +TEST_CASE("Create table success", "[createTable]") { + // REQUIRE(true); + CaseReporter cr("createTable"); + session->executeNonQueryStatement("DROP DATABASE IF EXISTS db1"); + session->executeNonQueryStatement("CREATE DATABASE db1"); + session->executeNonQueryStatement("DROP DATABASE IF EXISTS db2"); + session->executeNonQueryStatement("CREATE DATABASE db2"); + session->executeNonQueryStatement("USE \"db1\""); + session->executeNonQueryStatement("CREATE TABLE table0 (" + "tag1 string tag," + "attr1 string attribute," + "m1 double field)"); + unique_ptr sessionDataSet = session->executeQueryStatement("SHOW TABLES"); + sessionDataSet->setFetchSize(1024); + bool tableExist = false; + while (sessionDataSet->hasNext()) { + if (sessionDataSet->next()->fields[0].stringV == "table0") { + tableExist = true; + break; + } + } + REQUIRE(tableExist == true); +} + +TEST_CASE("Test insertRelationalTablet", "[testInsertRelationalTablet]") { + CaseReporter cr("testInsertRelationalTablet"); + session->executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + session->executeNonQueryStatement("USE db1"); + session->executeNonQueryStatement("DROP TABLE IF EXISTS table1"); + session->executeNonQueryStatement("CREATE TABLE table1 (" + "tag1 string tag," + "attr1 string attribute," + "m1 double field)"); + vector> schemaList; + schemaList.push_back(make_pair("tag1", TSDataType::TEXT)); + schemaList.push_back(make_pair("attr1", TSDataType::TEXT)); + schemaList.push_back(make_pair("m1", TSDataType::DOUBLE)); + vector columnTypes = {ColumnCategory::TAG, ColumnCategory::ATTRIBUTE, ColumnCategory::FIELD}; + + int64_t timestamp = 0; + Tablet tablet("table1", schemaList, columnTypes, 15); + + for (int row = 0; row < 15; row++) { + int rowIndex = tablet.rowSize++; + tablet.timestamps[rowIndex] = timestamp + row; + string data = "tag:"; data += to_string(row); + tablet.addValue(0, rowIndex, data); + data = "attr:"; data += to_string(row); + tablet.addValue(1, rowIndex, data); + double value = row * 1.1; + tablet.addValue(2, rowIndex, value); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insert(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session->insert(tablet, true); + tablet.reset(); + } + session->executeNonQueryStatement("FLUSH"); + + int cnt = 0; + unique_ptr sessionDataSet = session->executeQueryStatement("SELECT * FROM table1 order by time"); + while (sessionDataSet->hasNext()) { + RowRecord *rowRecord = sessionDataSet->next(); + REQUIRE(rowRecord->fields[1].stringV == string("tag:") + to_string(cnt)); + REQUIRE(rowRecord->fields[2].stringV == string("attr:") + to_string(cnt)); + REQUIRE(fabs(rowRecord->fields[3].doubleV - cnt * 1.1) < 0.0001); + cnt++; + } + REQUIRE(cnt == 15); +} diff --git a/iotdb-client/client-cpp/src/test/main.cpp b/iotdb-client/client-cpp/src/test/main.cpp index 9476343384c5a..82b3e60ddb593 100644 --- a/iotdb-client/client-cpp/src/test/main.cpp +++ b/iotdb-client/client-cpp/src/test/main.cpp @@ -22,7 +22,7 @@ #include #include "Session.h" -Session *session = new Session("127.0.0.1", 6667, "root", "root"); +std::shared_ptr session = std::make_shared("127.0.0.1", 6667, "root", "root"); struct SessionListener : Catch::TestEventListenerBase { diff --git a/iotdb-client/client-cpp/src/test/main_Relational.cpp b/iotdb-client/client-cpp/src/test/main_Relational.cpp new file mode 100644 index 0000000000000..ab028f0306af6 --- /dev/null +++ b/iotdb-client/client-cpp/src/test/main_Relational.cpp @@ -0,0 +1,50 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#define CATCH_CONFIG_MAIN + +#include +#include "TableSessionBuilder.h" + +auto builder = std::unique_ptr(new TableSessionBuilder()); +std::shared_ptr session = + std::shared_ptr( + builder + ->host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->build() + ); + +struct SessionListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; + + void testCaseStarting(Catch::TestCaseInfo const& testInfo) override { + // Perform some setup before a test case is run + session->open(); + } + + void testCaseEnded(Catch::TestCaseStats const& testCaseStats) override { + // Tear-down after a test case is run + session->close(); + } +}; + +CATCH_REGISTER_LISTENER(SessionListener) \ No newline at end of file diff --git a/iotdb-client/client-py/README.md b/iotdb-client/client-py/README.md index ec0c5a65de991..2cdf23b5e12b0 100644 --- a/iotdb-client/client-py/README.md +++ b/iotdb-client/client-py/README.md @@ -47,9 +47,9 @@ You have to install thrift (>=0.14.1) before using the package. First, download the latest package: `pip3 install apache-iotdb` -You can get an example of using the package to read and write data at here: [Example](https://github.com/apache/iotdb/blob/master/client-py/SessionExample.py) +You can get an example of using the package to read and write data at here: [Example](https://github.com/apache/iotdb/blob/master/iotdb-client/client-py/SessionExample.py) -An example of aligned timeseries: [Aligned Timeseries Session Example](https://github.com/apache/iotdb/blob/master/client-py/SessionAlignedTimeseriesExample.py) +An example of aligned timeseries: [Aligned Timeseries Session Example](https://github.com/apache/iotdb/blob/master/iotdb-client/client-py/SessionAlignedTimeseriesExample.py) (you need to add `import iotdb` in the head of the file) @@ -73,7 +73,7 @@ session.close() * Initialize a Session ```python -session = Session(ip, port_, username_, password_, fetch_size=1024, zone_id="UTC+8") +session = Session(ip, port_, username_, password_, fetch_size=1024, zone_id="Asia/Shanghai") ``` * Open a session, with a parameter to specify whether to enable RPC compression @@ -375,7 +375,7 @@ ip = "127.0.0.1" port_ = "6667" username_ = "root" password_ = "root" -conn = connect(ip, port_, username_, password_,fetch_size=1024,zone_id="UTC+8",sqlalchemy_mode=False) +conn = connect(ip, port_, username_, password_,fetch_size=1024,zone_id="Asia/Shanghai",sqlalchemy_mode=False) cursor = conn.cursor() ``` + simple SQL statement execution diff --git a/iotdb-client/client-py/SessionAlignedTimeseriesExample.py b/iotdb-client/client-py/SessionAlignedTimeseriesExample.py deleted file mode 100644 index 99b9f16a62ebc..0000000000000 --- a/iotdb-client/client-py/SessionAlignedTimeseriesExample.py +++ /dev/null @@ -1,225 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Uncomment the following line to use apache-iotdb module installed by pip3 - -from iotdb.Session import Session -from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor -from iotdb.utils.Tablet import Tablet - -# creating session connection. -ip = "127.0.0.1" -port_ = "6667" -username_ = "root" -password_ = "root" -session = Session(ip, port_, username_, password_, fetch_size=1024, zone_id="UTC+8") -session.open(False) - -# set and delete databases -session.set_storage_group("root.sg_test_01") -session.set_storage_group("root.sg_test_02") -session.set_storage_group("root.sg_test_03") -session.set_storage_group("root.sg_test_04") -session.delete_storage_group("root.sg_test_02") -session.delete_storage_groups(["root.sg_test_03", "root.sg_test_04"]) - -# setting aligned time series. -measurements_lst_ = [ - "s_01", - "s_02", - "s_03", -] -data_type_lst_ = [ - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64, -] -encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] -compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] -session.create_aligned_time_series( - "root.sg_test_01.d_02", - measurements_lst_, - data_type_lst_, - encoding_lst_, - compressor_lst_, -) - -# setting more aligned time series once. -measurements_lst_ = [ - "s_04", - "s_05", - "s_06", - "s_07", - "s_08", - "s_09", -] -data_type_lst_ = [ - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, -] -encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] -compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] -session.create_aligned_time_series( - "root.sg_test_01.d_02", - measurements_lst_, - data_type_lst_, - encoding_lst_, - compressor_lst_, -) - -# delete time series -session.delete_time_series( - [ - "root.sg_test_01.d_02.s_07", - "root.sg_test_01.d_02.s_08", - "root.sg_test_01.d_02.s_09", - ] -) - -# checking time series -print( - "s_07 expecting False, checking result: ", - session.check_time_series_exists("root.sg_test_01.d_02.s_07"), -) -print( - "s_03 expecting True, checking result: ", - session.check_time_series_exists("root.sg_test_01.d_02.s_03"), -) - -# insert one aligned record into the database. -measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] -values_ = [False, 10, 11, 1.1, 10011.1, "test_record"] -data_types_ = [ - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, -] -session.insert_aligned_record( - "root.sg_test_01.d_02", 1, measurements_, data_types_, values_ -) - -# insert multiple aligned records into database -measurements_list_ = [ - ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], - ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], -] -values_list_ = [ - [False, 22, 33, 4.4, 55.1, "test_records01"], - [True, 77, 88, 1.25, 8.125, "test_records02"], -] -data_type_list_ = [data_types_, data_types_] -device_ids_ = ["root.sg_test_01.d_02", "root.sg_test_01.d_02"] -session.insert_aligned_records( - device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ -) - -# insert one aligned tablet into the database. -values_ = [ - [False, 10, 11, 1.1, 10011.1, "test01"], - [True, 100, 11111, 1.25, 101.0, "test02"], - [False, 100, 1, 188.1, 688.25, "test03"], - [True, 0, 0, 0, 6.25, "test04"], -] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. -timestamps_ = [4, 5, 6, 7] -tablet_ = Tablet( - "root.sg_test_01.d_02", measurements_, data_types_, values_, timestamps_ -) -session.insert_aligned_tablet(tablet_) - -# insert multiple aligned tablets into database -tablet_01 = Tablet( - "root.sg_test_01.d_02", measurements_, data_types_, values_, [8, 9, 10, 11] -) -tablet_02 = Tablet( - "root.sg_test_01.d_02", measurements_, data_types_, values_, [12, 13, 14, 15] -) -session.insert_aligned_tablets([tablet_01, tablet_02]) - -# insert one aligned tablet with empty cells into the database. -values_ = [ - [None, 10, 11, 1.1, 10011.1, "test01"], - [True, None, 11111, 1.25, 101.0, "test02"], - [False, 100, 1, None, 688.25, "test03"], - [True, 0, 0, 0, 6.25, None], -] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. -timestamps_ = [16, 17, 18, 19] -tablet_ = Tablet( - "root.sg_test_01.d_02", measurements_, data_types_, values_, timestamps_ -) -session.insert_aligned_tablet(tablet_) - -# insert aligned records of one device -time_list = [1, 2, 3] -measurements_list = [ - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], -] -data_types_list = [ - [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], - [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], - [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], -] -values_list = [[False, 22, 33], [True, 1, 23], [False, 15, 26]] - -session.insert_aligned_records_of_one_device( - "root.sg_test_01.d_02", time_list, measurements_list, data_types_list, values_list -) - -# execute non-query sql statement -session.execute_non_query_statement( - "insert into root.sg_test_01.d_02(timestamp, s_02) aligned values(16, 188)" -) - -# execute sql query statement -with session.execute_query_statement( - "select * from root.sg_test_01.d_02" -) as session_data_set: - session_data_set.set_fetch_size(1024) - while session_data_set.has_next(): - print(session_data_set.next()) - -# insert aligned string record into the database. -time_list = [1, 2, 3] -measurements_list = [ - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], -] -values_list = [["False", "22", "33"], ["True", "1", "23"], ["False", "15", "26"]] -session.insert_aligned_string_records_of_one_device( - "root.sg_test_01.d_04", - time_list, - measurements_list, - values_list, -) - -# delete database -session.delete_storage_group("root.sg_test_01") - -# close session connection. -session.close() - -print("All executions done!!") diff --git a/iotdb-client/client-py/SessionExample.py b/iotdb-client/client-py/SessionExample.py deleted file mode 100644 index c489fe7d03d14..0000000000000 --- a/iotdb-client/client-py/SessionExample.py +++ /dev/null @@ -1,369 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Uncomment the following line to use apache-iotdb module installed by pip3 -import numpy as np - -from iotdb.Session import Session -from iotdb.utils.BitMap import BitMap -from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor -from iotdb.utils.Tablet import Tablet -from iotdb.utils.NumpyTablet import NumpyTablet - -# creating session connection. -ip = "127.0.0.1" -port_ = "6667" -username_ = "root" -password_ = "root" -# session = Session(ip, port_, username_, password_, fetch_size=1024, zone_id="UTC+8", enable_redirection=True) -session = Session.init_from_node_urls( - node_urls=["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"], - user="root", - password="root", - fetch_size=1024, - zone_id="UTC+8", - enable_redirection=True, -) -session.open(False) - -# create and delete databases -session.set_storage_group("root.sg_test_01") -session.set_storage_group("root.sg_test_02") -session.set_storage_group("root.sg_test_03") -session.set_storage_group("root.sg_test_04") -session.delete_storage_group("root.sg_test_02") -session.delete_storage_groups(["root.sg_test_03", "root.sg_test_04"]) - -# setting time series. -session.create_time_series( - "root.sg_test_01.d_01.s_01", TSDataType.BOOLEAN, TSEncoding.PLAIN, Compressor.SNAPPY -) -session.create_time_series( - "root.sg_test_01.d_01.s_02", TSDataType.INT32, TSEncoding.PLAIN, Compressor.SNAPPY -) -session.create_time_series( - "root.sg_test_01.d_01.s_03", TSDataType.INT64, TSEncoding.PLAIN, Compressor.SNAPPY -) -session.create_time_series( - "root.sg_test_01.d_02.s_01", - TSDataType.BOOLEAN, - TSEncoding.PLAIN, - Compressor.SNAPPY, - None, - {"tag1": "v1"}, - {"description": "v1"}, - "temperature", -) - -# setting multiple time series once. -ts_path_lst_ = [ - "root.sg_test_01.d_01.s_04", - "root.sg_test_01.d_01.s_05", - "root.sg_test_01.d_01.s_06", - "root.sg_test_01.d_01.s_07", - "root.sg_test_01.d_01.s_08", - "root.sg_test_01.d_01.s_09", -] -data_type_lst_ = [ - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, -] -encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] -compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] -session.create_multi_time_series( - ts_path_lst_, data_type_lst_, encoding_lst_, compressor_lst_ -) - -ts_path_lst_ = [ - "root.sg_test_01.d_02.s_04", - "root.sg_test_01.d_02.s_05", - "root.sg_test_01.d_02.s_06", - "root.sg_test_01.d_02.s_07", - "root.sg_test_01.d_02.s_08", - "root.sg_test_01.d_02.s_09", -] -data_type_lst_ = [ - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, -] -encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] -compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] -tags_lst_ = [{"tag2": "v2"} for _ in range(len(data_type_lst_))] -attributes_lst_ = [{"description": "v2"} for _ in range(len(data_type_lst_))] -session.create_multi_time_series( - ts_path_lst_, - data_type_lst_, - encoding_lst_, - compressor_lst_, - None, - tags_lst_, - attributes_lst_, - None, -) - -# delete time series -session.delete_time_series( - [ - "root.sg_test_01.d_01.s_07", - "root.sg_test_01.d_01.s_08", - "root.sg_test_01.d_01.s_09", - ] -) - -# checking time series -print( - "s_07 expecting False, checking result: ", - session.check_time_series_exists("root.sg_test_01.d_01.s_07"), -) -print( - "s_03 expecting True, checking result: ", - session.check_time_series_exists("root.sg_test_01.d_01.s_03"), -) -print( - "d_02.s_01 expecting True, checking result: ", - session.check_time_series_exists("root.sg_test_01.d_02.s_01"), -) -print( - "d_02.s_06 expecting True, checking result: ", - session.check_time_series_exists("root.sg_test_01.d_02.s_06"), -) - -# insert one record into the database. -measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] -values_ = [False, 10, 11, 1.1, 10011.1, "test_record"] -data_types_ = [ - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.TEXT, -] -session.insert_record("root.sg_test_01.d_01", 1, measurements_, data_types_, values_) - -# insert multiple records into database -measurements_list_ = [ - ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], - ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], -] -values_list_ = [ - [False, 22, 33, 4.4, 55.1, "test_records01"], - [True, 77, 88, 1.25, 8.125, bytes("test_records02", "utf-8")], -] -data_type_list_ = [data_types_, data_types_] -device_ids_ = ["root.sg_test_01.d_01", "root.sg_test_01.d_01"] -session.insert_records( - device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ -) - -# insert one tablet into the database. -values_ = [ - [False, 10, 11, 1.1, 10011.1, "test01"], - [True, 100, 11111, 1.25, 101.0, "test02"], - [False, 100, 1, 188.1, 688.25, "test03"], - [True, 0, 0, 0, 6.25, "test04"], -] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. -timestamps_ = [4, 5, 6, 7] -tablet_ = Tablet( - "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ -) -session.insert_tablet(tablet_) - -# insert one numpy tablet into the database. -np_values_ = [ - np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), - np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), - np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), - np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), - np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), - np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), -] -np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) -np_tablet_ = NumpyTablet( - "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ -) -session.insert_tablet(np_tablet_) - -# insert one unsorted numpy tablet into the database. -np_values_unsorted = [ - np.array([False, False, False, True, True], np.dtype(">?")), - np.array([0, 10, 100, 1000, 10000], np.dtype(">i4")), - np.array([1, 11, 111, 1111, 11111], np.dtype(">i8")), - np.array([1.1, 1.25, 188.1, 0, 8.999], np.dtype(">f4")), - np.array([10011.1, 101.0, 688.25, 6.25, 8, 776], np.dtype(">f8")), - np.array(["test09", "test08", "test07", "test06", "test05"]), -] -np_timestamps_unsorted = np.array([9, 8, 7, 6, 5], np.dtype(">i8")) -np_tablet_unsorted = NumpyTablet( - "root.sg_test_01.d_02", - measurements_, - data_types_, - np_values_unsorted, - np_timestamps_unsorted, -) - -# insert one numpy tablet into the database. -np_values_ = [ - np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), - np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), - np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), - np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), - np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), - np.array(["test01", "test02", "test03", "test04"]), -] -np_timestamps_ = np.array([98, 99, 100, 101], TSDataType.INT64.np_dtype()) -np_bitmaps_ = [] -for i in range(len(measurements_)): - np_bitmaps_.append(BitMap(len(np_timestamps_))) -np_bitmaps_[0].mark(0) -np_bitmaps_[1].mark(1) -np_bitmaps_[2].mark(2) -np_bitmaps_[4].mark(3) -np_bitmaps_[5].mark(3) -np_tablet_with_none = NumpyTablet( - "root.sg_test_01.d_02", - measurements_, - data_types_, - np_values_, - np_timestamps_, - np_bitmaps_, -) -session.insert_tablet(np_tablet_with_none) - - -session.insert_tablet(np_tablet_unsorted) -print(np_tablet_unsorted.get_timestamps()) -for value in np_tablet_unsorted.get_values(): - print(value) - -# insert multiple tablets into database -tablet_01 = Tablet( - "root.sg_test_01.d_01", measurements_, data_types_, values_, [8, 9, 10, 11] -) -tablet_02 = Tablet( - "root.sg_test_01.d_01", measurements_, data_types_, values_, [12, 13, 14, 15] -) -session.insert_tablets([tablet_01, tablet_02]) - -# insert one tablet with empty cells into the database. -values_ = [ - [None, 10, 11, 1.1, 10011.1, "test01"], - [True, None, 11111, 1.25, 101.0, "test02"], - [False, 100, 1, None, 688.25, "test03"], - [True, 0, 0, 0, 6.25, None], -] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. -timestamps_ = [16, 17, 18, 19] -tablet_ = Tablet( - "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ -) -session.insert_tablet(tablet_) - -# insert records of one device -time_list = [1, 2, 3] -measurements_list = [ - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], -] -data_types_list = [ - [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], - [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], - [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], -] -values_list = [[False, 22, 33], [True, 1, 23], [False, 15, 26]] - -session.insert_records_of_one_device( - "root.sg_test_01.d_01", time_list, measurements_list, data_types_list, values_list -) - -# execute non-query sql statement -session.execute_non_query_statement( - "insert into root.sg_test_01.d_01(timestamp, s_02) values(16, 188)" -) - -# execute sql query statement -with session.execute_query_statement( - "select * from root.sg_test_01.d_01" -) as session_data_set: - session_data_set.set_fetch_size(1024) - while session_data_set.has_next(): - print(session_data_set.next()) -# execute sql query statement -with session.execute_query_statement( - "select s_01, s_02, s_03, s_04, s_05, s_06 from root.sg_test_01.d_02" -) as session_data_set: - session_data_set.set_fetch_size(1024) - while session_data_set.has_next(): - print(session_data_set.next()) - -# execute statement -with session.execute_statement( - "select * from root.sg_test_01.d_01" -) as session_data_set: - while session_data_set.has_next(): - print(session_data_set.next()) - -session.execute_statement( - "insert into root.sg_test_01.d_01(timestamp, s_02) values(16, 188)" -) - -# insert string records of one device -time_list = [1, 2, 3] -measurements_list = [ - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], - ["s_01", "s_02", "s_03"], -] -values_list = [["False", "22", "33"], ["True", "1", "23"], ["False", "15", "26"]] - -session.insert_string_records_of_one_device( - "root.sg_test_01.d_03", - time_list, - measurements_list, - values_list, -) - -with session.execute_raw_data_query( - ["root.sg_test_01.d_03.s_01", "root.sg_test_01.d_03.s_02"], 1, 4 -) as session_data_set: - session_data_set.set_fetch_size(1024) - while session_data_set.has_next(): - print(session_data_set.next()) - -with session.execute_last_data_query( - ["root.sg_test_01.d_03.s_01", "root.sg_test_01.d_03.s_02"], 0 -) as session_data_set: - session_data_set.set_fetch_size(1024) - while session_data_set.has_next(): - print(session_data_set.next()) - -# delete database -session.delete_storage_group("root.sg_test_01") - -# close session connection. -session.close() - -print("All executions done!!") diff --git a/iotdb-client/client-py/SessionPoolExample.py b/iotdb-client/client-py/SessionPoolExample.py deleted file mode 100644 index 9230634ba6f90..0000000000000 --- a/iotdb-client/client-py/SessionPoolExample.py +++ /dev/null @@ -1,146 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import threading - -from iotdb.SessionPool import PoolConfig, SessionPool -from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor - -STORAGE_GROUP_NAME = "root.test" - - -def prepare_data(): - print("create database") - # Get a session from the pool - session = session_pool.get_session() - - session.set_storage_group(STORAGE_GROUP_NAME) - session.create_time_series( - "root.test.d0.s0", TSDataType.BOOLEAN, TSEncoding.PLAIN, Compressor.SNAPPY - ) - session.create_time_series( - "root.test.d0.s1", TSDataType.INT32, TSEncoding.PLAIN, Compressor.SNAPPY - ) - session.create_time_series( - "root.test.d0.s2", TSDataType.INT64, TSEncoding.PLAIN, Compressor.SNAPPY - ) - - # create multi time series - # setting multiple time series once. - ts_path_lst = [ - "root.test.d1.s0", - "root.test.d1.s1", - "root.test.d1.s2", - "root.test.d1.s3", - ] - data_type_lst = [ - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64, - TSDataType.FLOAT, - ] - encoding_lst = [TSEncoding.PLAIN for _ in range(len(data_type_lst))] - compressor_lst = [Compressor.SNAPPY for _ in range(len(data_type_lst))] - session.create_multi_time_series( - ts_path_lst, data_type_lst, encoding_lst, compressor_lst - ) - - # delete time series root.test.d1.s3 - session.delete_time_series(["root.test.d1.s3"]) - - print("now the timeseries are:") - # show result - res = session.execute_query_statement("show timeseries root.test.**") - while res.has_next(): - print(res.next()) - - session_pool.put_back(session) - - -def insert_data(num: int): - print("insert data for root.test.d" + str(num)) - device = ["root.test.d" + str(num)] - measurements = ["s0", "s1", "s2"] - values = [False, 10, 11] - data_types = [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64] - # Get a session from the pool - session = session_pool.get_session() - session.insert_records(device, [1], [measurements], [data_types], [values]) - - session_pool.put_back(session) - - -def query_data(): - # Get a session from the pool - session = session_pool.get_session() - - print("get data from root.test.d0") - res = session.execute_query_statement("select * from root.test.d0") - while res.has_next(): - print(res.next()) - - print("get data from root.test.d1") - res = session.execute_query_statement("select * from root.test.d1") - while res.has_next(): - print(res.next()) - - session_pool.put_back(session) - - -def delete_data(): - session = session_pool.get_session() - session.delete_storage_group(STORAGE_GROUP_NAME) - print("data has been deleted. now the devices are:") - res = session.execute_statement("show devices root.test.**") - while res.has_next(): - print(res.next()) - session_pool.put_back(session) - - -ip = "127.0.0.1" -port = "6667" -username = "root" -password = "root" -pool_config = PoolConfig( - node_urls=["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"], - user_name=username, - password=password, - fetch_size=1024, - time_zone="UTC+8", - max_retry=3, -) -max_pool_size = 5 -wait_timeout_in_ms = 3000 - -# Create a session pool -session_pool = SessionPool(pool_config, max_pool_size, wait_timeout_in_ms) - -prepare_data() - -insert_thread1 = threading.Thread(target=insert_data, args=(0,)) -insert_thread2 = threading.Thread(target=insert_data, args=(1,)) - -insert_thread1.start() -insert_thread2.start() - -insert_thread1.join() -insert_thread2.join() - -query_data() -delete_data() -session_pool.close() -print("example is finished!") diff --git a/iotdb-client/client-py/iotdb/Session.py b/iotdb-client/client-py/iotdb/Session.py index 5aca06c2781df..3f44bd41584d6 100644 --- a/iotdb-client/client-py/iotdb/Session.py +++ b/iotdb-client/client-py/iotdb/Session.py @@ -18,16 +18,17 @@ import logging import random +import sys import struct -import time import warnings from thrift.protocol import TBinaryProtocol, TCompactProtocol from thrift.transport import TSocket, TTransport +from tzlocal import get_localzone_name from iotdb.utils.SessionDataSet import SessionDataSet from .template.Template import Template from .template.TemplateQueryType import TemplateQueryType -from .thrift.common.ttypes import TEndPoint, TSStatus +from .thrift.common.ttypes import TEndPoint from .thrift.rpc.IClientRPCService import ( Client, TSCreateTimeseriesReq, @@ -58,21 +59,21 @@ TSLastDataQueryReq, TSInsertStringRecordsOfOneDeviceReq, ) -from .utils.IoTDBConnectionException import IoTDBConnectionException +from .tsfile.utils.date_utils import parse_date_to_int +from .utils import rpc_utils +from .utils.exception import IoTDBConnectionException, RedirectException logger = logging.getLogger("IoTDB") warnings.simplefilter("always", DeprecationWarning) class Session(object): - SUCCESS_STATUS = 200 - MULTIPLE_ERROR = 302 - REDIRECTION_RECOMMEND = 400 - DEFAULT_FETCH_SIZE = 10000 + DEFAULT_FETCH_SIZE = 5000 DEFAULT_USER = "root" DEFAULT_PASSWORD = "root" - DEFAULT_ZONE_ID = time.strftime("%z") + DEFAULT_ZONE_ID = get_localzone_name() RETRY_NUM = 3 + SQL_DIALECT = "tree" def __init__( self, @@ -83,6 +84,9 @@ def __init__( fetch_size=DEFAULT_FETCH_SIZE, zone_id=DEFAULT_ZONE_ID, enable_redirection=True, + use_ssl=False, + ca_certs=None, + connection_timeout_in_ms=None, ): self.__host = host self.__port = port @@ -103,6 +107,12 @@ def __init__( self.__enable_redirection = enable_redirection self.__device_id_to_endpoint = None self.__endpoint_to_connection = None + self.sql_dialect = self.SQL_DIALECT + self.database = None + self.__use_ssl = use_ssl + self.__ca_certs = ca_certs + self.__connection_timeout_in_ms = connection_timeout_in_ms + self.__time_precision = "ms" @classmethod def init_from_node_urls( @@ -113,11 +123,23 @@ def init_from_node_urls( fetch_size=DEFAULT_FETCH_SIZE, zone_id=DEFAULT_ZONE_ID, enable_redirection=True, + use_ssl=False, + ca_certs=None, + connection_timeout_in_ms=None, ): if node_urls is None: raise RuntimeError("node urls is empty") session = Session( - None, None, user, password, fetch_size, zone_id, enable_redirection + None, + None, + user, + password, + fetch_size, + zone_id, + enable_redirection, + use_ssl=use_ssl, + ca_certs=ca_certs, + connection_timeout_in_ms=connection_timeout_in_ms, ) session.__hosts = [] session.__ports = [] @@ -165,32 +187,31 @@ def open(self, enable_rpc_compression=False): } def init_connection(self, endpoint): - transport = TTransport.TFramedTransport( - TSocket.TSocket(endpoint.ip, endpoint.port) - ) - - if not transport.isOpen(): - try: - transport.open() - except TTransport.TTransportException as e: - raise IoTDBConnectionException(e) from None - + transport = self.__get_transport(endpoint) if self.__enable_rpc_compression: client = Client(TCompactProtocol.TCompactProtocolAccelerated(transport)) else: client = Client(TBinaryProtocol.TBinaryProtocolAccelerated(transport)) + configuration = {"version": "V_1_0", "sql_dialect": self.sql_dialect} + if self.database is not None: + configuration["db"] = self.database open_req = TSOpenSessionReq( client_protocol=self.protocol_version, username=self.__user, password=self.__password, zoneId=self.__zone_id, - configuration={"version": "V_1_0"}, + configuration=configuration, ) try: open_resp = client.openSession(open_req) - Session.verify_success(open_resp.status) + rpc_utils.verify_success(open_resp.status) + if open_resp.configuration is not None: + if "timestamp_precision" in open_resp.configuration: + self.__time_precision = open_resp.configuration[ + "timestamp_precision" + ] if self.protocol_version != open_resp.serverProtocolVersion: logger.exception( @@ -221,6 +242,33 @@ def init_connection(self, endpoint): self.__zone_id = self.get_time_zone() return SessionConnection(client, transport, session_id, statement_id) + def __get_transport(self, endpoint): + if self.__use_ssl: + import ssl + from thrift.transport import TSSLSocket + + if sys.version_info >= (3, 10): + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + else: + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(cafile=self.__ca_certs) + socket = TSSLSocket.TSSLSocket( + host=endpoint.ip, port=endpoint.port, ssl_context=context + ) + else: + socket = TSocket.TSocket(endpoint.ip, endpoint.port) + socket.setTimeout(self.__connection_timeout_in_ms) + transport = TTransport.TFramedTransport(socket) + + if not transport.isOpen(): + try: + transport.open() + except TTransport.TTransportException as e: + raise IoTDBConnectionException(e) from None + return transport + def is_open(self): return not self.__is_close @@ -244,13 +292,13 @@ def set_storage_group(self, group_name): :param group_name: String, database name (starts from root) """ try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.setStorageGroup(self.__session_id, group_name) ) except TTransport.TException as e: if self.reconnect(): try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.setStorageGroup(self.__session_id, group_name) ) except TTransport.TException as e1: @@ -272,13 +320,13 @@ def delete_storage_groups(self, storage_group_lst): :param storage_group_lst: List, paths of the target databases. """ try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.deleteStorageGroups(self.__session_id, storage_group_lst) ) except TTransport.TException as e: if self.reconnect(): try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.deleteStorageGroups( self.__session_id, storage_group_lst ) @@ -322,12 +370,12 @@ def create_time_series( alias, ) try: - return Session.verify_success(self.__client.createTimeseries(request)) + return rpc_utils.verify_success(self.__client.createTimeseries(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.createTimeseries(request) ) except TTransport.TException as e1: @@ -356,14 +404,14 @@ def create_aligned_time_series( compressor_lst, ) try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.createAlignedTimeseries(request) ) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.createAlignedTimeseries(request) ) except TTransport.TException as e1: @@ -406,12 +454,14 @@ def create_multi_time_series( alias_lst, ) try: - return Session.verify_success(self.__client.createMultiTimeseries(request)) + return rpc_utils.verify_success( + self.__client.createMultiTimeseries(request) + ) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.createMultiTimeseries(request) ) except TTransport.TException as e1: @@ -425,13 +475,13 @@ def delete_time_series(self, paths_list): :param paths_list: List of time series path, which should be complete (starts from root) """ try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.deleteTimeseries(self.__session_id, paths_list) ) except TTransport.TException as e: if self.reconnect(): try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.deleteTimeseries(self.__session_id, paths_list) ) except TTransport.TException as e1: @@ -460,12 +510,12 @@ def delete_data(self, paths_list, end_time): self.__session_id, paths_list, -9223372036854775808, end_time ) try: - return Session.verify_success(self.__client.deleteData(request)) + return rpc_utils.verify_success(self.__client.deleteData(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success(self.__client.deleteData(request)) + return rpc_utils.verify_success(self.__client.deleteData(request)) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -480,12 +530,12 @@ def delete_data_in_range(self, paths_list, start_time, end_time): """ request = TSDeleteDataReq(self.__session_id, paths_list, start_time, end_time) try: - return Session.verify_success(self.__client.deleteData(request)) + return rpc_utils.verify_success(self.__client.deleteData(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success(self.__client.deleteData(request)) + return rpc_utils.verify_success(self.__client.deleteData(request)) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -497,13 +547,22 @@ def insert_str_record(self, device_id, timestamp, measurements, string_values): string_values = [string_values] if type(measurements) == str: measurements = [measurements] + if self.__has_none_value(string_values): + filtered_measurements, filtered_values = zip( + *[(m, v) for m, v in zip(measurements, string_values) if v is not None] + ) + measurements = list(filtered_measurements) + values = list(filtered_values) + if len(measurements) == 0 or len(values) == 0: + logger.info("All inserting values are none!") + return request = self.gen_insert_str_record_req( device_id, timestamp, measurements, string_values ) try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertStringRecord(request) ) except RedirectException as e: @@ -512,7 +571,7 @@ def insert_str_record(self, device_id, timestamp, measurements, string_values): if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertStringRecord(request) ) except TTransport.TException as e1: @@ -528,13 +587,22 @@ def insert_aligned_str_record( string_values = [string_values] if type(measurements) == str: measurements = [measurements] + if self.__has_none_value(string_values): + filtered_measurements, filtered_values = zip( + *[(m, v) for m, v in zip(measurements, string_values) if v is not None] + ) + measurements = list(filtered_measurements) + values = list(filtered_values) + if len(measurements) == 0 or len(values) == 0: + logger.info("All inserting values are none!") + return request = self.gen_insert_str_record_req( device_id, timestamp, measurements, string_values, True ) try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertStringRecord(request) ) except RedirectException as e: @@ -543,7 +611,7 @@ def insert_aligned_str_record( if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertStringRecord(request) ) except TTransport.TException as e1: @@ -563,13 +631,27 @@ def insert_record(self, device_id, timestamp, measurements, data_types, values): :param data_types: List of TSDataType, indicate the data type for each sensor :param values: List, values to be inserted, for each sensor """ + if self.__has_none_value(values): + filtered_measurements, filtered_data_types, filtered_values = zip( + *[ + (m, d, v) + for m, d, v in zip(measurements, data_types, values) + if v is not None + ] + ) + measurements = list(filtered_measurements) + data_types = list(filtered_data_types) + values = list(filtered_values) + if len(measurements) == 0 or len(data_types) == 0 or len(values) == 0: + logger.info("All inserting values are none!") + return request = self.gen_insert_record_req( device_id, timestamp, measurements, data_types, values ) try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertRecord(request) ) except RedirectException as e: @@ -578,7 +660,7 @@ def insert_record(self, device_id, timestamp, measurements, data_types, values): if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success(self.__client.insertRecord(request)) + return rpc_utils.verify_success(self.__client.insertRecord(request)) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -596,6 +678,19 @@ def insert_records( :param types_lst: 2-D List of TSDataType, each element of outer list indicates sensor data types of a device :param values_lst: 2-D List, values to be inserted, for each device """ + if self.__has_none_value(values_lst): + ( + device_ids, + times, + measurements_lst, + types_lst, + values_lst, + ) = self.__filter_lists_by_values( + device_ids, times, measurements_lst, types_lst, values_lst + ) + if len(device_ids) == 0: + logger.info("All inserting values are none!") + return if self.__enable_redirection: request_group = {} for i in range(len(device_ids)): @@ -612,7 +707,7 @@ def insert_records( ) for client, request in request_group.items(): try: - Session.verify_success_with_redirection_for_multi_devices( + rpc_utils.verify_success_with_redirection_for_multi_devices( client.insertRecords(request), request.prefixPaths ) except RedirectException as e: @@ -622,7 +717,9 @@ def insert_records( if self.reconnect(): try: request.sessionId = self.__session_id - Session.verify_success(self.__client.insertRecords(request)) + rpc_utils.verify_success( + self.__client.insertRecords(request) + ) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -636,12 +733,12 @@ def insert_records( device_ids, times, measurements_lst, types_lst, values_lst ) try: - return Session.verify_success(self.__client.insertRecords(request)) + return rpc_utils.verify_success(self.__client.insertRecords(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertRecords(request) ) except TTransport.TException as e1: @@ -665,13 +762,27 @@ def insert_aligned_record( :param data_types: List of TSDataType, indicate the data type for each sensor :param values: List, values to be inserted, for each sensor """ + if self.__has_none_value(values): + filtered_measurements, filtered_data_types, filtered_values = zip( + *[ + (m, d, v) + for m, d, v in zip(measurements, data_types, values) + if v is not None + ] + ) + measurements = list(filtered_measurements) + data_types = list(filtered_data_types) + values = list(filtered_values) + if len(measurements) == 0 or len(data_types) == 0 or len(values) == 0: + logger.info("All inserting values are none!") + return request = self.gen_insert_record_req( device_id, timestamp, measurements, data_types, values, True ) try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertRecord(request) ) except RedirectException as e: @@ -680,7 +791,7 @@ def insert_aligned_record( if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success(self.__client.insertRecord(request)) + return rpc_utils.verify_success(self.__client.insertRecord(request)) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -698,6 +809,19 @@ def insert_aligned_records( :param types_lst: 2-D List of TSDataType, each element of outer list indicates sensor data types of a device :param values_lst: 2-D List, values to be inserted, for each device """ + if self.__has_none_value(values_lst): + ( + device_ids, + times, + measurements_lst, + types_lst, + values_lst, + ) = self.__filter_lists_by_values( + device_ids, times, measurements_lst, types_lst, values_lst + ) + if len(device_ids) == 0: + logger.info("All inserting values are none!") + return if self.__enable_redirection: request_group = {} for i in range(len(device_ids)): @@ -714,7 +838,7 @@ def insert_aligned_records( ) for client, request in request_group.items(): try: - Session.verify_success_with_redirection_for_multi_devices( + rpc_utils.verify_success_with_redirection_for_multi_devices( client.insertRecords(request), request.prefixPaths ) except RedirectException as e: @@ -724,7 +848,9 @@ def insert_aligned_records( if self.reconnect(): try: request.sessionId = self.__session_id - Session.verify_success(self.__client.insertRecords(request)) + rpc_utils.verify_success( + self.__client.insertRecords(request) + ) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -738,12 +864,12 @@ def insert_aligned_records( device_ids, times, measurements_lst, types_lst, values_lst, True ) try: - return Session.verify_success(self.__client.insertRecords(request)) + return rpc_utils.verify_success(self.__client.insertRecords(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertRecords(request) ) except TTransport.TException as e1: @@ -769,11 +895,11 @@ def test_insert_record( device_id, timestamp, measurements, data_types, values ) try: - return Session.verify_success(self.__client.testInsertRecord(request)) + return rpc_utils.verify_success(self.__client.testInsertRecord(request)) except TTransport.TException as e: if self.reconnect(): try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.testInsertRecord(request) ) except TTransport.TException as e1: @@ -797,11 +923,11 @@ def test_insert_records( device_ids, times, measurements_lst, types_lst, values_lst ) try: - return Session.verify_success(self.__client.testInsertRecords(request)) + return rpc_utils.verify_success(self.__client.testInsertRecords(request)) except TTransport.TException as e: if self.reconnect(): try: - return Session.verify_success( + return rpc_utils.verify_success( self.__client.testInsertRecords(request) ) except TTransport.TException as e1: @@ -890,18 +1016,20 @@ def insert_tablet(self, tablet): """ request = self.gen_insert_tablet_req(tablet) try: - connection = self.get_connection(tablet.get_device_id()) + connection = self.get_connection(tablet.get_insert_target_name()) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertTablet(request) ) except RedirectException as e: - return self.handle_redirection(tablet.get_device_id(), e.redirect_node) + return self.handle_redirection( + tablet.get_insert_target_name(), e.redirect_node + ) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success(self.__client.insertTablet(request)) + return rpc_utils.verify_success(self.__client.insertTablet(request)) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -915,14 +1043,14 @@ def insert_tablets(self, tablet_lst): if self.__enable_redirection: request_group = {} for i in range(len(tablet_lst)): - connection = self.get_connection(tablet_lst[i].get_device_id()) + connection = self.get_connection(tablet_lst[i].get_insert_target_name()) request = request_group.setdefault( connection.client, TSInsertTabletsReq( connection.session_id, [], [], [], [], [], [], False ), ) - request.prefixPaths.append(tablet_lst[i].get_device_id()) + request.prefixPaths.append(tablet_lst[i].get_insert_target_name()) request.timestampsList.append(tablet_lst[i].get_binary_timestamps()) request.measurementsList.append(tablet_lst[i].get_measurements()) request.valuesList.append(tablet_lst[i].get_binary_values()) @@ -930,7 +1058,7 @@ def insert_tablets(self, tablet_lst): request.typesList.append(tablet_lst[i].get_data_types()) for client, request in request_group.items(): try: - Session.verify_success_with_redirection_for_multi_devices( + rpc_utils.verify_success_with_redirection_for_multi_devices( client.insertTablets(request), request.prefixPaths ) except RedirectException as e: @@ -940,7 +1068,9 @@ def insert_tablets(self, tablet_lst): if self.reconnect(): try: request.sessionId = self.__session_id - Session.verify_success(self.__client.insertTablets(request)) + rpc_utils.verify_success( + self.__client.insertTablets(request) + ) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -952,12 +1082,12 @@ def insert_tablets(self, tablet_lst): else: request = self.gen_insert_tablets_req(tablet_lst) try: - return Session.verify_success(self.__client.insertTablets(request)) + return rpc_utils.verify_success(self.__client.insertTablets(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertTablets(request) ) except TTransport.TException as e1: @@ -981,18 +1111,20 @@ def insert_aligned_tablet(self, tablet): """ request = self.gen_insert_tablet_req(tablet, True) try: - connection = self.get_connection(tablet.get_device_id()) + connection = self.get_connection(tablet.get_insert_target_name()) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertTablet(request) ) except RedirectException as e: - return self.handle_redirection(tablet.get_device_id(), e.redirect_node) + return self.handle_redirection( + tablet.get_insert_target_name(), e.redirect_node + ) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success(self.__client.insertTablet(request)) + return rpc_utils.verify_success(self.__client.insertTablet(request)) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -1006,14 +1138,14 @@ def insert_aligned_tablets(self, tablet_lst): if self.__enable_redirection: request_group = {} for i in range(len(tablet_lst)): - connection = self.get_connection(tablet_lst[i].get_device_id()) + connection = self.get_connection(tablet_lst[i].get_insert_target_name()) request = request_group.setdefault( connection.client, TSInsertTabletsReq( connection.session_id, [], [], [], [], [], [], True ), ) - request.prefixPaths.append(tablet_lst[i].get_device_id()) + request.prefixPaths.append(tablet_lst[i].get_insert_target_name()) request.timestampsList.append(tablet_lst[i].get_binary_timestamps()) request.measurementsList.append(tablet_lst[i].get_measurements()) request.valuesList.append(tablet_lst[i].get_binary_values()) @@ -1021,7 +1153,7 @@ def insert_aligned_tablets(self, tablet_lst): request.typesList.append(tablet_lst[i].get_data_types()) for client, request in request_group.items(): try: - Session.verify_success_with_redirection_for_multi_devices( + rpc_utils.verify_success_with_redirection_for_multi_devices( client.insertTablets(request), request.prefixPaths ) except RedirectException as e: @@ -1031,7 +1163,9 @@ def insert_aligned_tablets(self, tablet_lst): if self.reconnect(): try: request.sessionId = self.__session_id - Session.verify_success(self.__client.insertTablets(request)) + rpc_utils.verify_success( + self.__client.insertTablets(request) + ) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: @@ -1043,12 +1177,12 @@ def insert_aligned_tablets(self, tablet_lst): else: request = self.gen_insert_tablets_req(tablet_lst, True) try: - return Session.verify_success(self.__client.insertTablets(request)) + return rpc_utils.verify_success(self.__client.insertTablets(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertTablets(request) ) except TTransport.TException as e1: @@ -1058,6 +1192,36 @@ def insert_aligned_tablets(self, tablet_lst): self.connection_error_msg() ) from None + def insert_relational_tablet(self, tablet): + """ + insert one tablet, for example three column in the table1 can form a tablet: + timestamps, id1, attr1, m1 + 1, id:1, attr:1, 1.0 + 2, id:1, attr:1, 2.0 + 3, id:2, attr:2, 3.0 + :param tablet: a tablet specified above + """ + request = self.gen_insert_relational_tablet_req(tablet) + try: + connection = self.get_connection(tablet.get_insert_target_name()) + request.sessionId = connection.session_id + return rpc_utils.verify_success_with_redirection( + connection.client.insertTablet(request) + ) + except RedirectException as e: + return self.handle_redirection( + tablet.get_insert_target_name(), e.redirect_node + ) + except TTransport.TException as e: + if self.reconnect(): + try: + request.sessionId = self.__session_id + return rpc_utils.verify_success(self.__client.insertTablet(request)) + except TTransport.TException as e1: + raise IoTDBConnectionException(e1) from None + else: + raise IoTDBConnectionException(self.connection_error_msg()) from None + def insert_records_of_one_device( self, device_id, times_list, measurements_list, types_list, values_list ): @@ -1111,7 +1275,7 @@ def insert_records_of_one_device_sorted( try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertRecordsOfOneDevice(request) ) except RedirectException as e: @@ -1120,7 +1284,7 @@ def insert_records_of_one_device_sorted( if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertRecordsOfOneDevice(request) ) except TTransport.TException as e1: @@ -1183,7 +1347,7 @@ def insert_aligned_records_of_one_device_sorted( try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertRecordsOfOneDevice(request) ) except RedirectException as e: @@ -1192,7 +1356,7 @@ def insert_aligned_records_of_one_device_sorted( if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertRecordsOfOneDevice(request) ) except TTransport.TException as e1: @@ -1237,12 +1401,12 @@ def test_insert_tablet(self, tablet): """ request = self.gen_insert_tablet_req(tablet) try: - return Session.verify_success(self.__client.testInsertTablet(request)) + return rpc_utils.verify_success(self.__client.testInsertTablet(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.testInsertTablet(request) ) except TTransport.TException as e1: @@ -1258,12 +1422,12 @@ def test_insert_tablets(self, tablet_list): """ request = self.gen_insert_tablets_req(tablet_list) try: - return Session.verify_success(self.__client.testInsertTablets(request)) + return rpc_utils.verify_success(self.__client.testInsertTablets(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.testInsertTablets(request) ) except TTransport.TException as e1: @@ -1274,13 +1438,27 @@ def test_insert_tablets(self, tablet_list): def gen_insert_tablet_req(self, tablet, is_aligned=False): return TSInsertTabletReq( self.__session_id, - tablet.get_device_id(), + tablet.get_insert_target_name(), + tablet.get_measurements(), + tablet.get_binary_values(), + tablet.get_binary_timestamps(), + tablet.get_data_types(), + tablet.get_row_number(), + is_aligned, + ) + + def gen_insert_relational_tablet_req(self, tablet, is_aligned=False): + return TSInsertTabletReq( + self.__session_id, + tablet.get_insert_target_name(), tablet.get_measurements(), tablet.get_binary_values(), tablet.get_binary_timestamps(), tablet.get_data_types(), tablet.get_row_number(), is_aligned, + True, + tablet.get_column_categories(), ) def gen_insert_tablets_req(self, tablet_lst, is_aligned=False): @@ -1291,7 +1469,7 @@ def gen_insert_tablets_req(self, tablet_lst, is_aligned=False): type_lst = [] size_lst = [] for tablet in tablet_lst: - device_id_lst.append(tablet.get_device_id()) + device_id_lst.append(tablet.get_insert_target_name()) measurements_lst.append(tablet.get_measurements()) values_lst.append(tablet.get_binary_values()) timestamps_lst.append(tablet.get_binary_timestamps()) @@ -1319,30 +1497,36 @@ def execute_query_statement(self, sql, timeout=0): self.__session_id, sql, self.__statement_id, self.__fetch_size, timeout ) try: - resp = self.__client.executeQueryStatement(request) + resp = self.__client.executeQueryStatementV2(request) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id request.statementId = self.__statement_id - resp = self.__client.executeQueryStatement(request) + resp = self.__client.executeQueryStatementV2(request) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: raise IoTDBConnectionException(self.connection_error_msg()) from None - Session.verify_success(resp.status) + rpc_utils.verify_success(resp.status) return SessionDataSet( sql, resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.queryId, + self.__session_id, self.__client, self.__statement_id, - self.__session_id, - resp.queryDataSet, + resp.queryResult, resp.ignoreTimeStamp, + timeout, + resp.moreData, + self.__fetch_size, + self.__zone_id, + self.__time_precision, + resp.columnIndex2TsBlockColumnIndexList, ) def execute_non_query_statement(self, sql): @@ -1352,38 +1536,50 @@ def execute_non_query_statement(self, sql): """ request = TSExecuteStatementReq(self.__session_id, sql, self.__statement_id) try: - resp = self.__client.executeUpdateStatement(request) + resp = self.__client.executeUpdateStatementV2(request) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id request.statementId = self.__statement_id - resp = self.__client.executeUpdateStatement(request) + resp = self.__client.executeUpdateStatementV2(request) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: raise IoTDBConnectionException(self.connection_error_msg()) from None - return Session.verify_success(resp.status) + previous_db = self.database + if resp.database is not None: + self.database = resp.database + if previous_db != self.database and self.__endpoint_to_connection is not None: + iterator = iter(self.__endpoint_to_connection.items()) + for entry in list(iterator): + endpoint, connection = entry + if connection != self.__default_connection: + try: + connection.change_database(sql) + except Exception as e: + self.__endpoint_to_connection.pop(endpoint) + return rpc_utils.verify_success(resp.status) def execute_statement(self, sql: str, timeout=0): request = TSExecuteStatementReq( self.__session_id, sql, self.__statement_id, timeout ) try: - resp = self.__client.executeStatement(request) + resp = self.__client.executeStatementV2(request) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id request.statementId = self.__statement_id - resp = self.__client.executeStatement(request) + resp = self.__client.executeStatementV2(request) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: raise IoTDBConnectionException(self.connection_error_msg()) from None - Session.verify_success(resp.status) + rpc_utils.verify_success(resp.status) if resp.columns: return SessionDataSet( sql, @@ -1391,11 +1587,17 @@ def execute_statement(self, sql: str, timeout=0): resp.dataTypeList, resp.columnNameIndexMap, resp.queryId, + self.__session_id, self.__client, self.__statement_id, - self.__session_id, - resp.queryDataSet, + resp.queryResult, resp.ignoreTimeStamp, + timeout, + resp.moreData, + self.__fetch_size, + self.__zone_id, + self.__time_precision, + resp.columnIndex2TsBlockColumnIndexList, ) else: return None @@ -1442,6 +1644,36 @@ def value_to_bytes(data_types, values): values_tobe_packed.append(b"\x05") values_tobe_packed.append(len(value_bytes)) values_tobe_packed.append(value_bytes) + # TIMESTAMP + elif data_type == 8: + format_str_list.append("cq") + values_tobe_packed.append(b"\x08") + values_tobe_packed.append(value) + # DATE + elif data_type == 9: + format_str_list.append("ci") + values_tobe_packed.append(b"\x09") + values_tobe_packed.append(parse_date_to_int(value)) + # BLOB + elif data_type == 10: + format_str_list.append("ci") + format_str_list.append(str(len(value))) + format_str_list.append("s") + values_tobe_packed.append(b"\x0a") + values_tobe_packed.append(len(value)) + values_tobe_packed.append(value) + # STRING + elif data_type == 11: + if isinstance(value, str): + value_bytes = bytes(value, "utf-8") + else: + value_bytes = value + format_str_list.append("ci") + format_str_list.append(str(len(value_bytes))) + format_str_list.append("s") + values_tobe_packed.append(b"\x0b") + values_tobe_packed.append(len(value_bytes)) + values_tobe_packed.append(value_bytes) else: raise RuntimeError("Unsupported data type:" + str(data_type)) format_str = "".join(format_str_list) @@ -1480,60 +1712,6 @@ def check_sorted(timestamps): return False return True - @staticmethod - def verify_success(status: TSStatus): - """ - verify success of operation - :param status: execution result status - """ - if status.code == Session.MULTIPLE_ERROR: - Session.verify_success_by_list(status.subStatus) - return 0 - if ( - status.code == Session.SUCCESS_STATUS - or status.code == Session.REDIRECTION_RECOMMEND - ): - return 0 - - raise RuntimeError(str(status.code) + ": " + status.message) - - @staticmethod - def verify_success_by_list(status_list: list): - """ - verify success of operation - :param status_list: execution result status - """ - message = str(Session.MULTIPLE_ERROR) + ": " - for status in status_list: - if ( - status.code != Session.SUCCESS_STATUS - and status.code != Session.REDIRECTION_RECOMMEND - ): - message += status.message + "; " - raise RuntimeError(message) - - @staticmethod - def verify_success_with_redirection(status: TSStatus): - Session.verify_success(status) - if status.redirectNode is not None: - raise RedirectException(status.redirectNode) - return 0 - - @staticmethod - def verify_success_with_redirection_for_multi_devices( - status: TSStatus, devices: list - ): - Session.verify_success(status) - if ( - status.code == Session.MULTIPLE_ERROR - or status.code == Session.REDIRECTION_RECOMMEND - ): - device_to_endpoint = {} - for i in range(len(status.subStatus)): - if status.subStatus[i].redirectNode is not None: - device_to_endpoint[devices[i]] = status.subStatus[i].redirectNode - raise RedirectException(device_to_endpoint) - def execute_raw_data_query( self, paths: list, start_time: int, end_time: int ) -> SessionDataSet: @@ -1554,29 +1732,35 @@ def execute_raw_data_query( enableRedirectQuery=False, ) try: - resp = self.__client.executeRawDataQuery(request) + resp = self.__client.executeRawDataQueryV2(request) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id request.statementId = self.__statement_id - resp = self.__client.executeRawDataQuery(request) + resp = self.__client.executeRawDataQueryV2(request) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: raise IoTDBConnectionException(self.connection_error_msg()) from None - Session.verify_success(resp.status) + rpc_utils.verify_success(resp.status) return SessionDataSet( "", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.queryId, + self.__session_id, self.__client, self.__statement_id, - self.__session_id, - resp.queryDataSet, + resp.queryResult, resp.ignoreTimeStamp, + 0, + resp.moreData, + self.__fetch_size, + self.__zone_id, + self.__time_precision, + resp.columnIndex2TsBlockColumnIndexList, ) def execute_last_data_query(self, paths: list, last_time: int) -> SessionDataSet: @@ -1595,29 +1779,35 @@ def execute_last_data_query(self, paths: list, last_time: int) -> SessionDataSet enableRedirectQuery=False, ) try: - resp = self.__client.executeLastDataQuery(request) + resp = self.__client.executeLastDataQueryV2(request) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id request.statementId = self.__statement_id - resp = self.__client.executeLastDataQuery(request) + resp = self.__client.executeLastDataQueryV2(request) except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None else: raise IoTDBConnectionException(self.connection_error_msg()) from None - Session.verify_success(resp.status) + rpc_utils.verify_success(resp.status) return SessionDataSet( "", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, resp.queryId, - self.__client, self.__statement_id, + self.__client, self.__session_id, - resp.queryDataSet, + resp.queryResult, resp.ignoreTimeStamp, + 0, + resp.moreData, + self.__fetch_size, + self.__zone_id, + self.__time_precision, + resp.columnIndex2TsBlockColumnIndexList, ) def insert_string_records_of_one_device( @@ -1648,7 +1838,7 @@ def insert_string_records_of_one_device( try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertStringRecordsOfOneDevice(request) ) except RedirectException as e: @@ -1657,7 +1847,7 @@ def insert_string_records_of_one_device( if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertStringRecordsOfOneDevice(request) ) except TTransport.TException as e1: @@ -1683,7 +1873,7 @@ def insert_aligned_string_records_of_one_device( try: connection = self.get_connection(device_id) request.sessionId = connection.session_id - return Session.verify_success_with_redirection( + return rpc_utils.verify_success_with_redirection( connection.client.insertStringRecordsOfOneDevice(request) ) except RedirectException as e: @@ -1692,7 +1882,7 @@ def insert_aligned_string_records_of_one_device( if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.insertStringRecordsOfOneDevice(request) ) except TTransport.TException as e1: @@ -1811,6 +2001,49 @@ def gen_insert_string_records_of_one_device_request( ) return request + def __has_none_value(self, values_list) -> bool: + for item in values_list: + if isinstance(item, list): + if self.__has_none_value(item): + return True + elif item is None: + return True + return False + + @staticmethod + def __filter_lists_by_values( + device_lst, time_lst, measurements_lst, types_lst, values_lst + ): + filtered_devices = [] + filtered_times = [] + filtered_measurements = [] + filtered_types = [] + filtered_values = [] + + for device, time_, measurements, types, values in zip( + device_lst, time_lst, measurements_lst, types_lst, values_lst + ): + filtered_row = [ + (m, t, v) + for m, t, v in zip(measurements, types, values) + if v is not None + ] + if filtered_row: + f_measurements, f_types, f_values = zip(*filtered_row) + filtered_measurements.append(list(f_measurements)) + filtered_types.append(list(f_types)) + filtered_values.append(list(f_values)) + filtered_devices.append(device) + filtered_times.append(time_) + + return ( + filtered_devices, + filtered_times, + filtered_measurements, + filtered_types, + filtered_values, + ) + def create_schema_template(self, template: Template): warnings.warn( "The APIs about template are deprecated and will be removed in future versions. Use sql instead.", @@ -1826,12 +2059,12 @@ def create_schema_template(self, template: Template): self.__session_id, template.get_name(), bytes_array ) try: - return Session.verify_success(self.__client.createSchemaTemplate(request)) + return rpc_utils.verify_success(self.__client.createSchemaTemplate(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.createSchemaTemplate(request) ) except TTransport.TException as e1: @@ -1851,12 +2084,12 @@ def drop_schema_template(self, template_name: str): """ request = TSDropSchemaTemplateReq(self.__session_id, template_name) try: - return Session.verify_success(self.__client.dropSchemaTemplate(request)) + return rpc_utils.verify_success(self.__client.dropSchemaTemplate(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.dropSchemaTemplate(request) ) except TTransport.TException as e1: @@ -1898,12 +2131,12 @@ def add_measurements_in_template( compressors, ) try: - return Session.verify_success(self.__client.appendSchemaTemplate(request)) + return rpc_utils.verify_success(self.__client.appendSchemaTemplate(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.appendSchemaTemplate(request) ) except TTransport.TException as e1: @@ -1924,12 +2157,12 @@ def delete_node_in_template(self, template_name: str, path: str): """ request = TSPruneSchemaTemplateReq(self.__session_id, template_name, path) try: - return Session.verify_success(self.__client.pruneSchemaTemplate(request)) + return rpc_utils.verify_success(self.__client.pruneSchemaTemplate(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.pruneSchemaTemplate(request) ) except TTransport.TException as e1: @@ -1950,12 +2183,12 @@ def set_schema_template(self, template_name, prefix_path): """ request = TSSetSchemaTemplateReq(self.__session_id, template_name, prefix_path) try: - return Session.verify_success(self.__client.setSchemaTemplate(request)) + return rpc_utils.verify_success(self.__client.setSchemaTemplate(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.setSchemaTemplate(request) ) except TTransport.TException as e1: @@ -1979,12 +2212,12 @@ def unset_schema_template(self, template_name, prefix_path): self.__session_id, prefix_path, template_name ) try: - return Session.verify_success(self.__client.unsetSchemaTemplate(request)) + return rpc_utils.verify_success(self.__client.unsetSchemaTemplate(request)) except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id - return Session.verify_success( + return rpc_utils.verify_success( self.__client.unsetSchemaTemplate(request) ) except TTransport.TException as e1: @@ -2015,7 +2248,7 @@ def count_measurements_in_template(self, template_name: str): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.count except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2041,14 +2274,14 @@ def is_measurement_in_template(self, template_name: str, path: str): ) try: response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.result except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.result except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2071,14 +2304,14 @@ def is_path_exist_in_template(self, template_name: str, path: str): ) try: response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.result except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.result except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2104,14 +2337,14 @@ def show_measurements_in_template(self, template_name: str, pattern: str = ""): ) try: response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2134,14 +2367,14 @@ def show_all_templates(self): ) try: response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2163,14 +2396,14 @@ def show_paths_template_set_on(self, template_name): ) try: response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2194,14 +2427,14 @@ def show_paths_template_using_on(self, template_name): ) try: response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e: if self.reconnect(): try: request.sessionId = self.__session_id response = self.__client.querySchemaTemplate(request) - Session.verify_success(response.status) + rpc_utils.verify_success(response.status) return response.measurements except TTransport.TException as e1: raise IoTDBConnectionException(e1) from None @@ -2222,6 +2455,17 @@ def __init__( self.session_id = session_id self.statement_id = statement_id + def change_database(self, sql): + try: + self.client.executeUpdateStatementV2( + TSExecuteStatementReq(self.session_id, sql, self.statement_id) + ) + except TTransport.TException as e: + raise IoTDBConnectionException( + "failed to change database", + e, + ) from None + def close_connection(self, req): try: self.client.closeSession(req) @@ -2233,12 +2477,3 @@ def close_connection(self, req): finally: if self.transport is not None: self.transport.close() - - -class RedirectException(Exception): - def __init__(self, redirect_info): - Exception.__init__(self) - if isinstance(redirect_info, TEndPoint): - self.redirect_node = redirect_info - else: - self.device_to_endpoint = redirect_info diff --git a/iotdb-client/client-py/iotdb/SessionPool.py b/iotdb-client/client-py/iotdb/SessionPool.py index 6f1d758079691..1e34186b2e0ca 100644 --- a/iotdb-client/client-py/iotdb/SessionPool.py +++ b/iotdb-client/client-py/iotdb/SessionPool.py @@ -21,12 +21,15 @@ from queue import Queue from threading import Lock +from tzlocal import get_localzone_name + from iotdb.Session import Session DEFAULT_MULTIPIE = 5 DEFAULT_FETCH_SIZE = 5000 DEFAULT_MAX_RETRY = 3 -DEFAULT_TIME_ZONE = "UTC+8" +DEFAULT_TIME_ZONE = get_localzone_name() +SQL_DIALECT = "tree" logger = logging.getLogger("IoTDB") @@ -42,6 +45,10 @@ def __init__( time_zone: str = DEFAULT_TIME_ZONE, max_retry: int = DEFAULT_MAX_RETRY, enable_compression: bool = False, + enable_redirection: bool = True, + use_ssl: bool = False, + ca_certs: str = None, + connection_timeout_in_ms: int = None, ): self.host = host self.port = port @@ -58,6 +65,10 @@ def __init__( self.time_zone = time_zone self.max_retry = max_retry self.enable_compression = enable_compression + self.enable_redirection = enable_redirection + self.use_ssl = use_ssl + self.ca_certs = ca_certs + self.connection_timeout_in_ms = connection_timeout_in_ms class SessionPool(object): @@ -71,6 +82,8 @@ def __init__( self.__queue = Queue(max_pool_size) self.__lock = Lock() self.__closed = False + self.sql_dialect = SQL_DIALECT + self.database = None def __construct_session(self) -> Session: if len(self.__pool_config.node_urls) > 0: @@ -80,7 +93,13 @@ def __construct_session(self) -> Session: self.__pool_config.password, self.__pool_config.fetch_size, self.__pool_config.time_zone, + enable_redirection=self.__pool_config.enable_redirection, + use_ssl=self.__pool_config.use_ssl, + ca_certs=self.__pool_config.ca_certs, + connection_timeout_in_ms=self.__pool_config.connection_timeout_in_ms, ) + session.sql_dialect = self.sql_dialect + session.database = self.database else: session = Session( @@ -90,7 +109,13 @@ def __construct_session(self) -> Session: self.__pool_config.password, self.__pool_config.fetch_size, self.__pool_config.time_zone, + enable_redirection=self.__pool_config.enable_redirection, + use_ssl=self.__pool_config.use_ssl, + ca_certs=self.__pool_config.ca_certs, + connection_timeout_in_ms=self.__pool_config.connection_timeout_in_ms, ) + session.sql_dialect = self.sql_dialect + session.database = self.database session.open(self.__pool_config.enable_compression) return session diff --git a/iotdb-client/client-py/iotdb/sqlalchemy/IoTDBDialect.py b/iotdb-client/client-py/iotdb/sqlalchemy/IoTDBDialect.py index 3d4ee192ad039..912e23e9f7a14 100644 --- a/iotdb-client/client-py/iotdb/sqlalchemy/IoTDBDialect.py +++ b/iotdb-client/client-py/iotdb/sqlalchemy/IoTDBDialect.py @@ -18,13 +18,14 @@ from sqlalchemy import types, util from sqlalchemy.engine import default +from sqlalchemy.sql import text from sqlalchemy.sql.sqltypes import String from iotdb import dbapi +from .IoTDBIdentifierPreparer import IoTDBIdentifierPreparer from .IoTDBSQLCompiler import IoTDBSQLCompiler from .IoTDBTypeCompiler import IoTDBTypeCompiler -from .IoTDBIdentifierPreparer import IoTDBIdentifierPreparer TYPES_MAP = { "BOOLEAN": types.Boolean, @@ -68,6 +69,10 @@ def create_connect_args(self, url): opts.update({"sqlalchemy_mode": True}) return [[], opts] + @classmethod + def import_dbapi(cls): + return dbapi + @classmethod def dbapi(cls): return dbapi @@ -79,17 +84,19 @@ def has_table(self, connection, table_name, schema=None, **kw): return table_name in self.get_table_names(connection, schema=schema) def get_schema_names(self, connection, **kw): - cursor = connection.execute("SHOW DATABASES") + cursor = connection.execute(text("SHOW DATABASES")) return [row[0] for row in cursor.fetchall()] def get_table_names(self, connection, schema=None, **kw): cursor = connection.execute( - "SHOW DEVICES %s.**" % (schema or self.default_schema_name) + text("SHOW DEVICES %s.**" % (schema or self.default_schema_name)) ) return [row[0].replace(schema + ".", "", 1) for row in cursor.fetchall()] def get_columns(self, connection, table_name, schema=None, **kw): - cursor = connection.execute("SHOW TIMESERIES %s.%s.*" % (schema, table_name)) + cursor = connection.execute( + text("SHOW TIMESERIES %s.%s.*" % (schema, table_name)) + ) columns = [self._general_time_column_info()] for row in cursor.fetchall(): columns.append(self._create_column_info(row, schema, table_name)) diff --git a/iotdb-client/client-py/iotdb/sqlalchemy/tests/test_dialect.py b/iotdb-client/client-py/iotdb/sqlalchemy/tests/test_dialect.py deleted file mode 100644 index cd6fae15bb1c2..0000000000000 --- a/iotdb-client/client-py/iotdb/sqlalchemy/tests/test_dialect.py +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import operator - -from sqlalchemy import create_engine, inspect -from sqlalchemy.dialects import registry - -from iotdb.IoTDBContainer import IoTDBContainer - -final_flag = True -failed_count = 0 - - -def test_fail(): - global failed_count - global final_flag - final_flag = False - failed_count += 1 - - -def print_message(message): - print("*********") - print(message) - print("*********") - assert False - - -def test_dialect(): - with IoTDBContainer("iotdb:dev") as db: - db: IoTDBContainer - url = ( - "iotdb://root:root@" - + db.get_container_host_ip() - + ":" - + db.get_exposed_port(6667) - ) - registry.register("iotdb", "iotdb.sqlalchemy.IoTDBDialect", "IoTDBDialect") - eng = create_engine(url) - eng.execute("create database root.cursor") - eng.execute("create database root.cursor_s1") - eng.execute( - "create timeseries root.cursor.device1.temperature with datatype=FLOAT,encoding=RLE" - ) - eng.execute( - "create timeseries root.cursor.device1.status with datatype=FLOAT,encoding=RLE" - ) - eng.execute( - "create timeseries root.cursor.device2.temperature with datatype=FLOAT,encoding=RLE" - ) - insp = inspect(eng) - # test get_schema_names - schema_names = insp.get_schema_names() - if not operator.ge(schema_names, ["root.cursor_s1", "root.cursor"]): - test_fail() - print_message("test get_schema_names failed!") - # test get_table_names - table_names = insp.get_table_names("root.cursor") - if not operator.eq(table_names, ["device1", "device2"]): - test_fail() - print_message("test get_table_names failed!") - # test get_columns - columns = insp.get_columns(table_name="device1", schema="root.cursor") - if len(columns) != 3: - test_fail() - print_message("test get_columns failed!") - eng.execute("delete database root.cursor") - eng.execute("delete database root.cursor_s1") - # close engine - eng.dispose() - - -if final_flag: - print("All executions done!!") -else: - print("Some test failed, please have a check") - print("failed count: ", failed_count) - exit(1) diff --git a/iotdb-client/client-py/iotdb/table_session.py b/iotdb-client/client-py/iotdb/table_session.py new file mode 100644 index 0000000000000..fdf20e7cfa596 --- /dev/null +++ b/iotdb-client/client-py/iotdb/table_session.py @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from typing import Union + +from iotdb.Session import Session +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.SessionDataSet import SessionDataSet +from iotdb.utils.Tablet import Tablet + + +class TableSessionConfig(object): + + def __init__( + self, + node_urls: list = None, + username: str = Session.DEFAULT_USER, + password: str = Session.DEFAULT_PASSWORD, + database: str = None, + fetch_size: int = 5000, + time_zone: str = Session.DEFAULT_ZONE_ID, + enable_redirection: bool = True, + enable_compression: bool = False, + use_ssl: bool = False, + ca_certs: str = None, + connection_timeout_in_ms: int = None, + ): + """ + Initialize a TableSessionConfig object with the provided parameters. + + Parameters: + node_urls (list, optional): A list of node URLs for the database connection. + Defaults to ["localhost:6667"]. + username (str, optional): The username for the database connection. + Defaults to "root". + password (str, optional): The password for the database connection. + Defaults to "root". + database (str, optional): The target database to connect to. Defaults to None. + fetch_size (int, optional): The number of rows to fetch per query. Defaults to 5000. + time_zone (str, optional): The default time zone for the session. + Defaults to Session.DEFAULT_ZONE_ID. + enable_redirection (bool, optional): Whether to enable redirection. + Defaults to False. + enable_compression (bool, optional): Whether to enable data compression. + Defaults to False. + + """ + if node_urls is None: + node_urls = ["localhost:6667"] + self.node_urls = node_urls + self.username = username + self.password = password + self.database = database + self.fetch_size = fetch_size + self.time_zone = time_zone + self.enable_redirection = enable_redirection + self.enable_compression = enable_compression + self.use_ssl = use_ssl + self.ca_certs = ca_certs + self.connection_timeout_in_ms = connection_timeout_in_ms + + +class TableSession(object): + + def __init__( + self, table_session_config: TableSessionConfig = None, session_pool=None + ): + self.__session_pool = session_pool + if self.__session_pool is None: + self.__session = Session.init_from_node_urls( + table_session_config.node_urls, + table_session_config.username, + table_session_config.password, + table_session_config.fetch_size, + table_session_config.time_zone, + table_session_config.enable_redirection, + table_session_config.use_ssl, + table_session_config.ca_certs, + table_session_config.connection_timeout_in_ms, + ) + self.__session.sql_dialect = "table" + self.__session.database = table_session_config.database + self.__session.open(table_session_config.enable_compression) + else: + self.__session = self.__session_pool.get_session() + + def insert(self, tablet: Union[Tablet, NumpyTablet]): + """ + Insert data into the database. + + Parameters: + tablet (Tablet | NumpyTablet): The tablet containing the data to be inserted. + Accepts either a `Tablet` or `NumpyTablet`. + + Raises: + IoTDBConnectionException: If there is an issue with the database connection. + """ + self.__session.insert_relational_tablet(tablet) + + def execute_non_query_statement(self, sql: str): + """ + Execute a non-query SQL statement. + + Parameters: + sql (str): The SQL statement to execute. Typically used for commands + such as INSERT, DELETE, or UPDATE. + + Raises: + IoTDBConnectionException: If there is an issue with the database connection. + """ + self.__session.execute_non_query_statement(sql) + + def execute_query_statement( + self, sql: str, timeout_in_ms: int = 0 + ) -> SessionDataSet: + """ + Execute a query SQL statement and return the result set. + + Parameters: + sql (str): The SQL query to execute. + timeout_in_ms (int, optional): Timeout for the query in milliseconds. Defaults to 0, + which means no timeout. + + Returns: + SessionDataSet: The result set of the query. + + Raises: + IoTDBConnectionException: If there is an issue with the database connection. + """ + return self.__session.execute_query_statement(sql, timeout_in_ms) + + def close(self): + """ + Close the session and release resources. + + Raises: + IoTDBConnectionException: If there is an issue closing the connection. + """ + if self.__session_pool is None: + self.__session.close() + else: + self.__session_pool.put_back(self.__session) diff --git a/iotdb-client/client-py/iotdb/table_session_pool.py b/iotdb-client/client-py/iotdb/table_session_pool.py new file mode 100644 index 0000000000000..f44df3f249b6b --- /dev/null +++ b/iotdb-client/client-py/iotdb/table_session_pool.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from iotdb.Session import Session +from iotdb.SessionPool import SessionPool, PoolConfig +from iotdb.table_session import TableSession + + +class TableSessionPoolConfig(object): + def __init__( + self, + node_urls: list = None, + max_pool_size: int = 5, + username: str = Session.DEFAULT_USER, + password: str = Session.DEFAULT_PASSWORD, + database: str = None, + fetch_size: int = 5000, + time_zone: str = Session.DEFAULT_ZONE_ID, + enable_redirection: bool = False, + enable_compression: bool = False, + wait_timeout_in_ms: int = 10000, + max_retry: int = 3, + use_ssl: bool = False, + ca_certs: str = None, + connection_timeout_in_ms: int = None, + ): + """ + Initialize a TableSessionPoolConfig object with the provided parameters. + + Parameters: + node_urls (list, optional): A list of node URLs for the database connection. + Defaults to None. + max_pool_size (int, optional): The maximum number of sessions in the pool. + Defaults to 5. + username (str, optional): The username for the database connection. + Defaults to Session.DEFAULT_USER. + password (str, optional): The password for the database connection. + Defaults to Session.DEFAULT_PASSWORD. + database (str, optional): The target database to connect to. Defaults to None. + fetch_size (int, optional): The number of rows to fetch per query. Defaults to 5000. + time_zone (str, optional): The default time zone for the session pool. + Defaults to Session.DEFAULT_ZONE_ID. + enable_redirection (bool, optional): Whether to enable redirection. + Defaults to False. + enable_compression (bool, optional): Whether to enable data compression. + Defaults to False. + wait_timeout_in_ms (int, optional): The maximum time (in milliseconds) to wait for a session + to become available. Defaults to 10000. + max_retry (int, optional): The maximum number of retry attempts for operations. Defaults to 3. + + """ + if node_urls is None: + node_urls = ["localhost:6667"] + self.pool_config = PoolConfig( + node_urls=node_urls, + user_name=username, + password=password, + fetch_size=fetch_size, + time_zone=time_zone, + max_retry=max_retry, + enable_redirection=enable_redirection, + enable_compression=enable_compression, + use_ssl=use_ssl, + ca_certs=ca_certs, + connection_timeout_in_ms=connection_timeout_in_ms, + ) + self.max_pool_size = max_pool_size + self.wait_timeout_in_ms = wait_timeout_in_ms + self.database = database + + +class TableSessionPool(object): + + def __init__(self, table_session_pool_config: TableSessionPoolConfig): + pool_config = table_session_pool_config.pool_config + max_pool_size = table_session_pool_config.max_pool_size + wait_timeout_in_ms = table_session_pool_config.wait_timeout_in_ms + self.__session_pool = SessionPool( + pool_config, max_pool_size, wait_timeout_in_ms + ) + self.__session_pool.sql_dialect = "table" + self.__session_pool.database = table_session_pool_config.database + + def get_session(self) -> TableSession: + """ + Retrieve a new TableSession instance. + + Returns: + TableSession: A new session object configured with the session pool. + + Notes: + The session is initialized with the underlying session pool for managing + connections. Ensure proper usage of the session's lifecycle. + """ + return TableSession(None, session_pool=self.__session_pool) + + def close(self): + """ + Close the session pool and release all resources. + + This method closes the underlying session pool, ensuring that all + resources associated with it are properly released. + + Notes: + After calling this method, the session pool cannot be used to retrieve + new sessions, and any attempt to do so may raise an exception. + """ + self.__session_pool.close() diff --git a/iotdb-client/client-py/iotdb/template/MeasurementNode.py b/iotdb-client/client-py/iotdb/template/MeasurementNode.py index b1c6f50217ddb..06cd7c6957e79 100644 --- a/iotdb-client/client-py/iotdb/template/MeasurementNode.py +++ b/iotdb-client/client-py/iotdb/template/MeasurementNode.py @@ -19,7 +19,7 @@ from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor from .TemplateNode import TemplateNode -from ..tsfile.utils.ReadWriteIOUtils import ReadWriteUtils +from ..tsfile.utils.read_write_io_utils import ReadWriteUtils warnings.simplefilter("always", DeprecationWarning) diff --git a/iotdb-client/client-py/iotdb/template/Template.py b/iotdb-client/client-py/iotdb/template/Template.py index 0c226c7cb6dee..98a6082f1483d 100644 --- a/iotdb-client/client-py/iotdb/template/Template.py +++ b/iotdb-client/client-py/iotdb/template/Template.py @@ -20,9 +20,9 @@ import warnings from .TemplateNode import TemplateNode -from ..tsfile.common.constant.TsFileConstant import TsFileConstant -from ..tsfile.utils.Pair import Pair -from ..tsfile.utils.ReadWriteIOUtils import ReadWriteUtils +from ..tsfile.common.constant.tsfile_constant import TsFileConstant +from ..tsfile.utils.pair import Pair +from ..tsfile.utils.read_write_io_utils import ReadWriteUtils warnings.simplefilter("always", DeprecationWarning) diff --git a/iotdb-client/client-py/iotdb/tsfile/common/constant/TsFileConstant.py b/iotdb-client/client-py/iotdb/tsfile/common/constant/tsfile_constant.py similarity index 100% rename from iotdb-client/client-py/iotdb/tsfile/common/constant/TsFileConstant.py rename to iotdb-client/client-py/iotdb/tsfile/common/constant/tsfile_constant.py diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/date_utils.py b/iotdb-client/client-py/iotdb/tsfile/utils/date_utils.py new file mode 100644 index 0000000000000..4cafedea6183b --- /dev/null +++ b/iotdb-client/client-py/iotdb/tsfile/utils/date_utils.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from datetime import date + + +class DateTimeParseException(Exception): + pass + + +def parse_int_to_date(date_int: int) -> date: + try: + year = date_int // 10000 + month = (date_int // 100) % 100 + day = date_int % 100 + return date(year, month, day) + except ValueError as e: + raise DateTimeParseException("Invalid date format.") from e + + +def parse_date_to_int(local_date: date) -> int: + if local_date is None: + raise DateTimeParseException("Date expression is none or empty.") + if local_date.year < 1000: + raise DateTimeParseException("Year must be between 1000 and 9999.") + return local_date.year * 10000 + local_date.month * 100 + local_date.day diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/Pair.py b/iotdb-client/client-py/iotdb/tsfile/utils/pair.py similarity index 100% rename from iotdb-client/client-py/iotdb/tsfile/utils/Pair.py rename to iotdb-client/client-py/iotdb/tsfile/utils/pair.py diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/ReadWriteIOUtils.py b/iotdb-client/client-py/iotdb/tsfile/utils/read_write_io_utils.py similarity index 100% rename from iotdb-client/client-py/iotdb/tsfile/utils/ReadWriteIOUtils.py rename to iotdb-client/client-py/iotdb/tsfile/utils/read_write_io_utils.py diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py b/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py new file mode 100644 index 0000000000000..854be120ffb52 --- /dev/null +++ b/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py @@ -0,0 +1,266 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np + + +# Serialized tsBlock: +# +-------------+---------------+---------+------------+-----------+----------+ +# | val col cnt | val col types | pos cnt | encodings | time col | val col | +# +-------------+---------------+---------+------------+-----------+----------+ +# | int32 | list[byte] | int32 | list[byte] | bytes | byte | +# +-------------+---------------+---------+------------+-----------+----------+ + + +def deserialize(buffer): + value_column_count, buffer = read_int_from_buffer(buffer) + data_types, buffer = read_column_types(buffer, value_column_count) + + position_count, buffer = read_int_from_buffer(buffer) + column_encodings, buffer = read_column_encoding(buffer, value_column_count + 1) + + time_column_values, _, buffer = read_column( + column_encodings[0], buffer, 2, position_count + ) + column_values = [] + null_indicators = [] + for i in range(value_column_count): + column_value, null_indicator, buffer = read_column( + column_encodings[i + 1], buffer, data_types[i], position_count + ) + column_values.append(column_value) + null_indicators.append(null_indicator) + + return time_column_values, column_values, null_indicators, position_count + + +# General Methods + + +def read_int_from_buffer(buffer): + res = np.frombuffer(buffer, dtype=">i4", count=1) + buffer = buffer[4:] + return res[0], buffer + + +def read_byte_from_buffer(buffer): + return read_from_buffer(buffer, 1) + + +def read_from_buffer(buffer, size): + res = buffer[:size] + new_buffer = buffer[size:] + return res, new_buffer + + +# Read ColumnType + + +def read_column_types(buffer, value_column_count): + data_types = np.frombuffer(buffer, dtype=np.uint8, count=value_column_count) + new_buffer = buffer[value_column_count:] + if not np.all(np.isin(data_types, (0, 1, 2, 3, 4, 5))): + raise Exception("Invalid data type encountered: " + str(data_types)) + return data_types, new_buffer + + +# Read ColumnEncodings + + +def read_column_encoding(buffer, size): + encodings = np.frombuffer(buffer, dtype=np.uint8, count=size) + new_buffer = buffer[size:] + return encodings, new_buffer + + +# Read Column + + +def deserialize_null_indicators(buffer, size): + may_have_null = buffer[0] + buffer = buffer[1:] + if may_have_null != 0: + return deserialize_from_boolean_array(buffer, size) + return None, buffer + + +# Serialized data layout: +# +---------------+-----------------+-------------+ +# | may have null | null indicators | values | +# +---------------+-----------------+-------------+ +# | byte | list[byte] | list[int64] | +# +---------------+-----------------+-------------+ + + +def read_int64_column(buffer, data_type, position_count): + null_indicators, buffer = deserialize_null_indicators(buffer, position_count) + if null_indicators is None: + size = position_count + else: + size = np.count_nonzero(~null_indicators) + + if data_type == 2: + dtype = ">i8" + elif data_type == 4: + dtype = ">f8" + else: + raise Exception("Invalid data type: " + str(data_type)) + values = np.frombuffer(buffer, dtype, count=size) + buffer = buffer[size * 8 :] + return values, null_indicators, buffer + + +# Serialized data layout: +# +---------------+-----------------+-------------+ +# | may have null | null indicators | values | +# +---------------+-----------------+-------------+ +# | byte | list[byte] | list[int32] | +# +---------------+-----------------+-------------+ + + +def read_int32_column(buffer, data_type, position_count): + null_indicators, buffer = deserialize_null_indicators(buffer, position_count) + if null_indicators is None: + size = position_count + else: + size = np.count_nonzero(~null_indicators) + + if data_type == 1: + dtype = ">i4" + elif data_type == 3: + dtype = ">f4" + else: + raise Exception("Invalid data type: " + str(data_type)) + values = np.frombuffer(buffer, dtype, count=size) + buffer = buffer[size * 4 :] + return values, null_indicators, buffer + + +# Serialized data layout: +# +---------------+-----------------+-------------+ +# | may have null | null indicators | values | +# +---------------+-----------------+-------------+ +# | byte | list[byte] | list[byte] | +# +---------------+-----------------+-------------+ + + +def read_byte_column(buffer, data_type, position_count): + if data_type != 0: + raise Exception("Invalid data type: " + data_type) + null_indicators, buffer = deserialize_null_indicators(buffer, position_count) + res, buffer = deserialize_from_boolean_array(buffer, position_count) + return res, null_indicators, buffer + + +def deserialize_from_boolean_array(buffer, size): + num_bytes = (size + 7) // 8 + packed_boolean_array, buffer = read_from_buffer(buffer, num_bytes) + arr = np.frombuffer(packed_boolean_array, dtype=np.uint8) + output = np.unpackbits(arr)[:size].astype(bool) + return output, buffer + + +# Serialized data layout: +# +---------------+-----------------+-------------+ +# | may have null | null indicators | values | +# +---------------+-----------------+-------------+ +# | byte | list[byte] | list[entry] | +# +---------------+-----------------+-------------+ +# +# Each entry is represented as: +# +---------------+-------+ +# | value length | value | +# +---------------+-------+ +# | int32 | bytes | +# +---------------+-------+ + + +def read_binary_column(buffer, data_type, position_count): + if data_type != 5: + raise Exception("Invalid data type: " + data_type) + null_indicators, buffer = deserialize_null_indicators(buffer, position_count) + + if null_indicators is None: + size = position_count + else: + size = np.count_nonzero(~null_indicators) + values = np.empty(size, dtype=object) + for i in range(size): + length, buffer = read_int_from_buffer(buffer) + res, buffer = read_from_buffer(buffer, length) + values[i] = res.tobytes() + return values, null_indicators, buffer + + +# Serialized data layout: +# +-----------+-------------------------+ +# | encoding | serialized inner column | +# +-----------+-------------------------+ +# | byte | list[byte] | +# +-----------+-------------------------+ + + +def read_run_length_column(buffer, data_type, position_count): + encoding, buffer = read_byte_from_buffer(buffer) + column, null_indicators, buffer = read_column(encoding[0], buffer, data_type, 1) + return ( + repeat(column, data_type, position_count), + ( + None + if null_indicators is None + else np.full(position_count, null_indicators[0]) + ), + buffer, + ) + + +def repeat(column, data_type, position_count): + if data_type in (0, 5): + if column.size == 1: + return np.full( + position_count, column[0], dtype=(bool if data_type == 0 else object) + ) + else: + return np.array(column * position_count, dtype=object) + else: + res = bytearray() + for _ in range(position_count): + res.extend(column if isinstance(column, bytes) else bytes(column)) + return bytes(res) + + +def read_dictionary_column(buffer, data_type, position_count): + raise Exception("dictionary column not implemented") + + +ENCODING_FUNC_MAP = { + 0: read_byte_column, + 1: read_int32_column, + 2: read_int64_column, + 3: read_binary_column, + 4: read_run_length_column, + 5: read_dictionary_column, +} + + +def read_column(encoding, buffer, data_type, position_count): + try: + func = ENCODING_FUNC_MAP[encoding] + except KeyError: + raise Exception("Unsupported encoding: " + str(encoding)) + return func(buffer, data_type, position_count) diff --git a/iotdb-client/client-py/iotdb/utils/Field.py b/iotdb-client/client-py/iotdb/utils/Field.py index 913c7594bf59c..d9a0ee77776ec 100644 --- a/iotdb-client/client-py/iotdb/utils/Field.py +++ b/iotdb-client/client-py/iotdb/utils/Field.py @@ -17,18 +17,22 @@ # # for package -from .IoTDBConstants import TSDataType +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.tsfile.utils.date_utils import parse_int_to_date +from iotdb.utils.rpc_utils import convert_to_timestamp, isoformat import numpy as np import pandas as pd class Field(object): - def __init__(self, data_type, value=None): + def __init__(self, data_type, value=None, timezone=None, precision=None): """ :param data_type: TSDataType """ self.__data_type = data_type self.value = value + self.__timezone = timezone + self.__precision = precision @staticmethod def copy(field): @@ -36,15 +40,25 @@ def copy(field): if output.get_data_type() is not None: if output.get_data_type() == TSDataType.BOOLEAN: output.set_bool_value(field.get_bool_value()) - elif output.get_data_type() == TSDataType.INT32: + elif ( + output.get_data_type() == TSDataType.INT32 + or output.get_data_type() == TSDataType.DATE + ): output.set_int_value(field.get_int_value()) - elif output.get_data_type() == TSDataType.INT64: + elif ( + output.get_data_type() == TSDataType.INT64 + or output.get_data_type() == TSDataType.TIMESTAMP + ): output.set_long_value(field.get_long_value()) elif output.get_data_type() == TSDataType.FLOAT: output.set_float_value(field.get_float_value()) elif output.get_data_type() == TSDataType.DOUBLE: output.set_double_value(field.get_double_value()) - elif output.get_data_type() == TSDataType.TEXT: + elif ( + output.get_data_type() == TSDataType.TEXT + or output.get_data_type() == TSDataType.STRING + or output.get_data_type() == TSDataType.BLOB + ): output.set_binary_value(field.get_binary_value()) else: raise Exception( @@ -80,6 +94,7 @@ def get_int_value(self): raise Exception("Null Field Exception!") if ( self.__data_type != TSDataType.INT32 + and self.__data_type != TSDataType.DATE or self.value is None or self.value is pd.NA ): @@ -94,6 +109,7 @@ def get_long_value(self): raise Exception("Null Field Exception!") if ( self.__data_type != TSDataType.INT64 + and self.__data_type != TSDataType.TIMESTAMP or self.value is None or self.value is pd.NA ): @@ -136,17 +152,52 @@ def get_binary_value(self): raise Exception("Null Field Exception!") if ( self.__data_type != TSDataType.TEXT + and self.__data_type != TSDataType.STRING + and self.__data_type != TSDataType.BLOB or self.value is None or self.value is pd.NA ): return None return self.value + def get_timestamp_value(self): + if self.__data_type is None: + raise Exception("Null Field Exception!") + if ( + self.__data_type != TSDataType.TIMESTAMP + or self.value is None + or self.value is pd.NA + ): + return None + return convert_to_timestamp(self.value, self.__precision, self.__timezone) + + def get_date_value(self): + if self.__data_type is None: + raise Exception("Null Field Exception!") + if ( + self.__data_type != TSDataType.DATE + or self.value is None + or self.value is pd.NA + ): + return None + return parse_int_to_date(self.value) + def get_string_value(self): if self.__data_type is None or self.value is None or self.value is pd.NA: return "None" - elif self.__data_type == 5: + # TEXT, STRING + if self.__data_type == 5 or self.__data_type == 11: return self.value.decode("utf-8") + # BLOB + elif self.__data_type == 10: + return str(hex(int.from_bytes(self.value, byteorder="big"))) + # TIMESTAMP + elif self.__data_type == 8: + return isoformat( + convert_to_timestamp(self.value, self.__precision, self.__timezone), + self.__precision, + ) + # Others else: return str(self.get_object_value(self.__data_type)) @@ -169,7 +220,16 @@ def get_object_value(self, data_type): return np.float32(self.value) elif data_type == 4: return np.float64(self.value) - return self.value + elif data_type == 8: + return convert_to_timestamp(self.value, self.__precision, self.__timezone) + elif data_type == 9: + return parse_int_to_date(self.value) + elif data_type == 5 or data_type == 11: + return self.value.decode("utf-8") + elif data_type == 10: + return self.value + else: + raise RuntimeError("Unsupported data type:" + str(data_type)) @staticmethod def get_field(value, data_type): diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConnectionException.py b/iotdb-client/client-py/iotdb/utils/IoTDBConnectionException.py deleted file mode 100644 index 0f7b895d1f8ae..0000000000000 --- a/iotdb-client/client-py/iotdb/utils/IoTDBConnectionException.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -class IoTDBConnectionException(Exception): - def __init__(self, reason=None, cause=None, message=None): - if reason is not None: - super().__init__(reason) - elif cause is not None: - super().__init__(cause) - elif message is not None and cause is not None: - super().__init__(message, cause) - else: - super().__init__() diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py index 9b671663ff4fa..4b9082b5353fd 100644 --- a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py +++ b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +from datetime import date from enum import unique, IntEnum import numpy as np @@ -27,6 +28,10 @@ class TSDataType(IntEnum): FLOAT = 3 DOUBLE = 4 TEXT = 5 + TIMESTAMP = 8 + DATE = 9 + BLOB = 10 + STRING = 11 def np_dtype(self): return { @@ -35,7 +40,11 @@ def np_dtype(self): TSDataType.DOUBLE: np.dtype(">f8"), TSDataType.INT32: np.dtype(">i4"), TSDataType.INT64: np.dtype(">i8"), - TSDataType.TEXT: np.dtype("str"), + TSDataType.TEXT: str, + TSDataType.TIMESTAMP: np.dtype(">i8"), + TSDataType.DATE: date, + TSDataType.BLOB: bytes, + TSDataType.STRING: str, }[self] diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py b/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py deleted file mode 100644 index 3151b2acfd8c5..0000000000000 --- a/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py +++ /dev/null @@ -1,413 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# for package -import binascii -import logging - -import numpy as np -import pandas as pd -from thrift.transport import TTransport -from iotdb.thrift.rpc.IClientRPCService import TSFetchResultsReq, TSCloseOperationReq -from iotdb.utils.IoTDBConstants import TSDataType - -logger = logging.getLogger("IoTDB") - - -def _to_bitbuffer(b): - return bytes("{:0{}b}".format(int(binascii.hexlify(b), 16), 8 * len(b)), "utf-8") - - -class IoTDBRpcDataSet(object): - TIMESTAMP_STR = "Time" - # VALUE_IS_NULL = "The value got by %s (column name) is NULL." - START_INDEX = 2 - - def __init__( - self, - sql, - column_name_list, - column_type_list, - column_name_index, - ignore_timestamp, - query_id, - client, - statement_id, - session_id, - query_data_set, - fetch_size, - ): - self.__statement_id = statement_id - self.__session_id = session_id - self.ignore_timestamp = ignore_timestamp - self.__sql = sql - self.__query_id = query_id - self.__client = client - self.__fetch_size = fetch_size - self.column_size = len(column_name_list) - self.__default_time_out = 1000 - - self.__column_name_list = [] - self.__column_type_list = [] - self.column_ordinal_dict = {} - if not ignore_timestamp: - self.__column_name_list.append(IoTDBRpcDataSet.TIMESTAMP_STR) - self.__column_type_list.append(TSDataType.INT64) - self.column_ordinal_dict[IoTDBRpcDataSet.TIMESTAMP_STR] = 1 - - if column_name_index is not None: - self.column_type_deduplicated_list = [ - None for _ in range(len(column_name_index)) - ] - for i in range(self.column_size): - name = column_name_list[i] - self.__column_name_list.append(name) - self.__column_type_list.append(TSDataType[column_type_list[i]]) - if name not in self.column_ordinal_dict: - index = column_name_index[name] - self.column_ordinal_dict[name] = index + IoTDBRpcDataSet.START_INDEX - self.column_type_deduplicated_list[index] = TSDataType[ - column_type_list[i] - ] - else: - index = IoTDBRpcDataSet.START_INDEX - self.column_type_deduplicated_list = [] - for i in range(len(column_name_list)): - name = column_name_list[i] - self.__column_name_list.append(name) - self.__column_type_list.append(TSDataType[column_type_list[i]]) - if name not in self.column_ordinal_dict: - self.column_ordinal_dict[name] = index - index += 1 - self.column_type_deduplicated_list.append( - TSDataType[column_type_list[i]] - ) - self.__query_data_set = query_data_set - self.__is_closed = False - self.__empty_resultSet = False - self.__rows_index = 0 - self.has_cached_data_frame = False - self.data_frame = None - - def close(self): - if self.__is_closed: - return - if self.__client is not None: - try: - status = self.__client.closeOperation( - TSCloseOperationReq( - self.__session_id, self.__query_id, self.__statement_id - ) - ) - logger.debug( - "close session {}, message: {}".format( - self.__session_id, status.message - ) - ) - except TTransport.TException as e: - raise RuntimeError( - "close session {} failed because: ".format(self.__session_id), e - ) - - self.__is_closed = True - self.__client = None - - def next(self): - if not self.has_cached_data_frame: - self.construct_one_data_frame() - if self.has_cached_data_frame: - return True - if self.__empty_resultSet: - return False - if self.fetch_results(): - self.construct_one_data_frame() - return True - return False - - def construct_one_data_frame(self): - if ( - self.has_cached_data_frame - or self.__query_data_set is None - or len(self.__query_data_set.time) == 0 - ): - return - result = {} - time_array = np.frombuffer( - self.__query_data_set.time, np.dtype(np.longlong).newbyteorder(">") - ) - if time_array.dtype.byteorder == ">": - time_array = time_array.byteswap().view(time_array.dtype.newbyteorder("<")) - result[0] = time_array - total_length = len(time_array) - for i in range(self.column_size): - if self.ignore_timestamp is True: - column_name = self.__column_name_list[i] - else: - column_name = self.__column_name_list[i + 1] - - location = ( - self.column_ordinal_dict[column_name] - IoTDBRpcDataSet.START_INDEX - ) - if location < 0: - continue - data_type = self.column_type_deduplicated_list[location] - value_buffer = self.__query_data_set.valueList[location] - value_buffer_len = len(value_buffer) - if data_type == 4: - data_array = np.frombuffer( - value_buffer, np.dtype(np.double).newbyteorder(">") - ) - elif data_type == 3: - data_array = np.frombuffer( - value_buffer, np.dtype(np.float32).newbyteorder(">") - ) - elif data_type == 0: - data_array = np.frombuffer(value_buffer, np.dtype("?")) - elif data_type == 1: - data_array = np.frombuffer( - value_buffer, np.dtype(np.int32).newbyteorder(">") - ) - elif data_type == 2: - data_array = np.frombuffer( - value_buffer, np.dtype(np.int64).newbyteorder(">") - ) - elif data_type == 5: - j = 0 - offset = 0 - data_array = [] - while offset < value_buffer_len: - length = int.from_bytes( - value_buffer[offset : offset + 4], - byteorder="big", - signed=False, - ) - offset += 4 - value = bytes(value_buffer[offset : offset + length]) - data_array.append(value) - j += 1 - offset += length - data_array = np.array(data_array, dtype=object) - else: - raise RuntimeError("unsupported data type {}.".format(data_type)) - if data_array.dtype.byteorder == ">": - data_array = data_array.byteswap().view( - data_array.dtype.newbyteorder("<") - ) - # self.__query_data_set.valueList[location] = None - if len(data_array) < total_length: - # INT32 or INT64 or boolean - if data_type == 0 or data_type == 1 or data_type == 2: - tmp_array = np.full(total_length, np.nan, np.float32) - else: - tmp_array = np.full(total_length, None, dtype=object) - - bitmap_buffer = self.__query_data_set.bitmapList[location] - buffer = _to_bitbuffer(bitmap_buffer) - bit_mask = (np.frombuffer(buffer, "u1") - ord("0")).astype(bool) - if len(bit_mask) != total_length: - bit_mask = bit_mask[:total_length] - tmp_array[bit_mask] = data_array - - if data_type == 1: - tmp_array = pd.Series(tmp_array, dtype="Int32") - elif data_type == 2: - tmp_array = pd.Series(tmp_array, dtype="Int64") - elif data_type == 0: - tmp_array = pd.Series(tmp_array, dtype="boolean") - data_array = tmp_array - - result[i + 1] = data_array - self.__query_data_set = None - self.data_frame = pd.DataFrame(result, dtype=object) - if not self.data_frame.empty: - self.has_cached_data_frame = True - - def has_cached_result(self): - return self.has_cached_data_frame - - def _has_next_result_set(self): - if (self.__query_data_set is not None) and ( - len(self.__query_data_set.time) != 0 - ): - return True - if self.__empty_resultSet: - return False - if self.fetch_results(): - return True - return False - - def resultset_to_pandas(self): - result = {} - for column_name in self.__column_name_list: - result[column_name] = [] - while self._has_next_result_set(): - time_array = np.frombuffer( - self.__query_data_set.time, np.dtype(np.longlong).newbyteorder(">") - ) - if time_array.dtype.byteorder == ">": - time_array = time_array.byteswap().view( - time_array.dtype.newbyteorder("<") - ) - if self.ignore_timestamp is None or self.ignore_timestamp is False: - result[IoTDBRpcDataSet.TIMESTAMP_STR].append(time_array) - - self.__query_data_set.time = [] - total_length = len(time_array) - - for i in range(len(self.__query_data_set.bitmapList)): - if self.ignore_timestamp is True: - column_name = self.__column_name_list[i] - else: - column_name = self.__column_name_list[i + 1] - - location = ( - self.column_ordinal_dict[column_name] - IoTDBRpcDataSet.START_INDEX - ) - if location < 0: - continue - data_type = self.column_type_deduplicated_list[location] - value_buffer = self.__query_data_set.valueList[location] - value_buffer_len = len(value_buffer) - if data_type == 4: - data_array = np.frombuffer( - value_buffer, np.dtype(np.double).newbyteorder(">") - ) - elif data_type == 3: - data_array = np.frombuffer( - value_buffer, np.dtype(np.float32).newbyteorder(">") - ) - elif data_type == 0: - data_array = np.frombuffer(value_buffer, np.dtype("?")) - elif data_type == 1: - data_array = np.frombuffer( - value_buffer, np.dtype(np.int32).newbyteorder(">") - ) - elif data_type == 2: - data_array = np.frombuffer( - value_buffer, np.dtype(np.int64).newbyteorder(">") - ) - elif data_type == 5: - j = 0 - offset = 0 - data_array = [] - while offset < value_buffer_len: - length = int.from_bytes( - value_buffer[offset : offset + 4], - byteorder="big", - signed=False, - ) - offset += 4 - value_bytes = bytes(value_buffer[offset : offset + length]) - value = value_bytes.decode("utf-8") - data_array.append(value) - j += 1 - offset += length - data_array = np.array(data_array, dtype=object) - else: - raise RuntimeError("unsupported data type {}.".format(data_type)) - if data_array.dtype.byteorder == ">": - data_array = data_array.byteswap().view( - data_array.dtype.newbyteorder("<") - ) - self.__query_data_set.valueList[location] = None - tmp_array = [] - if len(data_array) < total_length: - if data_type == 1 or data_type == 2: - tmp_array = np.full(total_length, np.nan, np.float32) - elif data_type == 3 or data_type == 4: - tmp_array = np.full(total_length, np.nan, data_array.dtype) - elif data_type == 0: - tmp_array = np.full(total_length, np.nan, np.float32) - elif data_type == 5: - tmp_array = np.full(total_length, None, dtype=data_array.dtype) - - bitmap_buffer = self.__query_data_set.bitmapList[location] - buffer = _to_bitbuffer(bitmap_buffer) - bit_mask = (np.frombuffer(buffer, "u1") - ord("0")).astype(bool) - if len(bit_mask) != total_length: - bit_mask = bit_mask[:total_length] - tmp_array[bit_mask] = data_array - - if data_type == 1: - tmp_array = pd.Series(tmp_array).astype("Int32") - elif data_type == 2: - tmp_array = pd.Series(tmp_array).astype("Int64") - elif data_type == 0: - tmp_array = pd.Series(tmp_array).astype("boolean") - - data_array = tmp_array - - result[column_name].append(data_array) - - for k, v in result.items(): - if v is None or len(v) < 1 or v[0] is None: - result[k] = [] - elif v[0].dtype == "Int32": - result[k] = pd.Series(np.concatenate(v, axis=0)).astype("Int32") - elif v[0].dtype == "Int64": - result[k] = pd.Series(np.concatenate(v, axis=0)).astype("Int64") - elif v[0].dtype == bool: - result[k] = pd.Series(np.concatenate(v, axis=0)).astype("boolean") - else: - result[k] = np.concatenate(v, axis=0) - - df = pd.DataFrame(result) - return df - - def fetch_results(self): - self.__rows_index = 0 - request = TSFetchResultsReq( - self.__session_id, - self.__sql, - self.__fetch_size, - self.__query_id, - True, - self.__default_time_out, - ) - try: - resp = self.__client.fetchResults(request) - if not resp.hasResultSet: - self.__empty_resultSet = True - else: - self.__query_data_set = resp.queryDataSet - return resp.hasResultSet - except TTransport.TException as e: - raise RuntimeError( - "Cannot fetch result from server, because of network connection: ", e - ) - - def find_column_name_by_index(self, column_index): - if column_index <= 0: - raise Exception("Column index should start from 1") - if column_index > len(self.__column_name_list): - raise Exception( - "column index {} out of range {}".format(column_index, self.column_size) - ) - return self.__column_name_list[column_index - 1] - - def get_fetch_size(self): - return self.__fetch_size - - def set_fetch_size(self, fetch_size): - self.__fetch_size = fetch_size - - def get_column_names(self): - return self.__column_name_list - - def get_column_types(self): - return self.__column_type_list diff --git a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py index 4577f7f880c42..9a0860d3a43e9 100644 --- a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py +++ b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py @@ -17,28 +17,48 @@ # import struct + +import numpy as np +from numpy import ndarray +from typing import List + +from iotdb.tsfile.utils.date_utils import parse_date_to_int from iotdb.utils.IoTDBConstants import TSDataType from iotdb.utils.BitMap import BitMap +from iotdb.utils.Tablet import ColumnType class NumpyTablet(object): def __init__( - self, device_id, measurements, data_types, values, timestamps, bitmaps=None + self, + insert_target_name: str, + column_names: List[str], + data_types: List[TSDataType], + values: List[ndarray], + timestamps: ndarray, + bitmaps: List[BitMap] = None, + column_types: List[ColumnType] = None, ): """ creating a numpy tablet for insertion - for example, considering device: root.sg1.d1 + for example using tree-model, considering device: root.sg1.d1 timestamps, m1, m2, m3 1, 125.3, True, text1 2, 111.6, False, text2 3, 688.6, True, text3 - Notice: From 0.13.0, the tablet can contain empty cell - The tablet will be sorted at the initialization by timestamps - :param device_id: String, IoTDB time series path to device layer (without sensor) - :param measurements: List, sensors - :param data_types: TSDataType List, specify value types for sensors - :param values: List of numpy array, the values of each column should be the inner numpy array - :param timestamps: Numpy array, the timestamps + for example using table-model, considering table: table1 + timestamps, id1, attr1, m1 + 1, id:1, attr:1, 1.0 + 2, id:1, attr:1, 2.0 + 3, id:2, attr:2, 3.0 + Notice: The tablet will be sorted at the initialization by timestamps + :param insert_target_name: Str, DeviceId if using tree model or TableName when using table model. + :param column_names: Str List, names of columns + :param data_types: TSDataType List, specify value types for columns + :param values: ndarray List, one ndarray contains the value of one column + :param timestamps: ndarray, contains the timestamps + :param bitmaps: BitMap list, one bitmap records the position of none value in a column + :param column_types: ColumnType List, marking the type of each column, can be none for tree-view interfaces. """ if len(values) > 0 and len(values[0]) != len(timestamps): raise RuntimeError( @@ -63,12 +83,18 @@ def __init__( self.__values = values self.__timestamps = timestamps - self.__device_id = device_id - self.__measurements = measurements + self.__insert_target_name = insert_target_name + self.__measurements = column_names self.__data_types = data_types self.__row_number = len(timestamps) - self.__column_number = len(measurements) + self.__column_number = len(column_names) self.bitmaps = bitmaps + if column_types is None: + self.__column_types = ColumnType.n_copy( + ColumnType.FIELD, self.__column_number + ) + else: + self.__column_types = column_types @staticmethod def check_sorted(timestamps): @@ -83,11 +109,14 @@ def get_measurements(self): def get_data_types(self): return self.__data_types + def get_column_categories(self): + return self.__column_types + def get_row_number(self): return self.__row_number - def get_device_id(self): - return self.__device_id + def get_insert_target_name(self): + return self.__insert_target_name def get_timestamps(self): return self.__timestamps @@ -102,12 +131,22 @@ def get_binary_values(self): bs_len = 0 bs_list = [] for data_type, value in zip(self.__data_types, self.__values): - # TEXT - if data_type == 5: + # BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + if ( + data_type == 0 + or data_type == 1 + or data_type == 2 + or data_type == 3 + or data_type == 4 + or data_type == 8 + ): + bs = value.tobytes() + # TEXT, STRING, BLOB + elif data_type == 5 or data_type == 11 or data_type == 10: format_str_list = [">"] values_tobe_packed = [] for str_list in value: - # Fot TEXT, it's same as the original solution + # For TEXT, it's same as the original solution if isinstance(str_list, str): value_bytes = bytes(str_list, "utf-8") else: @@ -119,11 +158,18 @@ def get_binary_values(self): values_tobe_packed.append(value_bytes) format_str = "".join(format_str_list) bs = struct.pack(format_str, *values_tobe_packed) - # Non-TEXT + # DATE + elif data_type == 9: + bs = ( + np.vectorize(parse_date_to_int)(value) + .astype(np.dtype(">i4")) + .tobytes() + ) else: - bs = value.tobytes() + raise RuntimeError("Unsupported data type:" + str(data_type)) bs_list.append(bs) bs_len += len(bs) + if self.bitmaps is not None: format_str_list = [">"] values_tobe_packed = [] diff --git a/iotdb-client/client-py/iotdb/utils/RowRecord.py b/iotdb-client/client-py/iotdb/utils/RowRecord.py index 16a88f1edf9b1..91bcebf948b0d 100644 --- a/iotdb-client/client-py/iotdb/utils/RowRecord.py +++ b/iotdb-client/client-py/iotdb/utils/RowRecord.py @@ -17,15 +17,11 @@ # # for package -from .Field import Field - -# for debug -# from IoTDBConstants import TSDataType -# from Field import Field +from iotdb.utils.Field import Field class RowRecord(object): - def __init__(self, timestamp, field_list=None): + def __init__(self, timestamp, field_list: list = None): self.__timestamp = timestamp self.__field_list = field_list diff --git a/iotdb-client/client-py/iotdb/utils/SessionDataSet.py b/iotdb-client/client-py/iotdb/utils/SessionDataSet.py index a7ce7dfc40cb3..b079ee28c1aaf 100644 --- a/iotdb-client/client-py/iotdb/utils/SessionDataSet.py +++ b/iotdb-client/client-py/iotdb/utils/SessionDataSet.py @@ -21,7 +21,7 @@ # for package from iotdb.utils.IoTDBConstants import TSDataType -from iotdb.utils.IoTDBRpcDataSet import IoTDBRpcDataSet +from iotdb.utils.iotdb_rpc_dataset import IoTDBRpcDataSet from iotdb.utils.RowRecord import RowRecord import pandas as pd @@ -37,40 +37,51 @@ def __init__( column_type_list, column_name_index, query_id, - client, statement_id, + client, session_id, - query_data_set, + query_result, ignore_timestamp, + time_out, + more_data, + fetch_size, + zone_id, + time_precision, + column_index_2_tsblock_column_index_list, ): self.iotdb_rpc_data_set = IoTDBRpcDataSet( sql, column_name_list, column_type_list, - column_name_index, ignore_timestamp, + more_data, query_id, client, statement_id, session_id, - query_data_set, - 5000, - ) - self.column_size = self.iotdb_rpc_data_set.column_size - self.is_ignore_timestamp = self.iotdb_rpc_data_set.ignore_timestamp - self.column_names = tuple(self.iotdb_rpc_data_set.get_column_names()) - self.column_ordinal_dict = self.iotdb_rpc_data_set.column_ordinal_dict - self.column_type_deduplicated_list = tuple( - self.iotdb_rpc_data_set.column_type_deduplicated_list + query_result, + fetch_size, + time_out, + zone_id, + time_precision, + column_index_2_tsblock_column_index_list, ) - if self.is_ignore_timestamp: + if ignore_timestamp: self.__field_list = [ - Field(data_type) + ( + Field(data_type, timezone=zone_id, precision=time_precision) + if data_type == 8 + else Field(data_type) + ) for data_type in self.iotdb_rpc_data_set.get_column_types() ] else: self.__field_list = [ - Field(data_type) + ( + Field(data_type, timezone=zone_id, precision=time_precision) + if data_type == 8 + else Field(data_type) + ) for data_type in self.iotdb_rpc_data_set.get_column_types()[1:] ] self.row_index = 0 @@ -105,14 +116,22 @@ def next(self): def construct_row_record_from_data_frame(self): df = self.iotdb_rpc_data_set.data_frame row = df.iloc[self.row_index].to_list() - row_values = row[1:] - for i, value in enumerate(row_values): - self.__field_list[i].value = value + if self.iotdb_rpc_data_set.ignore_timestamp: + for field, value in zip(self.__field_list, row): + field.value = value + + row_record = RowRecord( + 0, + self.__field_list, + ) + else: + for field, value in zip(self.__field_list, row[1:]): + field.value = value - row_record = RowRecord( - row[0], - self.__field_list, - ) + row_record = RowRecord( + row[0], + self.__field_list, + ) self.row_index += 1 if self.row_index == len(df): self.row_index = 0 @@ -124,29 +143,33 @@ def construct_row_record_from_data_frame(self): def close_operation_handle(self): self.iotdb_rpc_data_set.close() - def todf(self): - return resultset_to_pandas(self) + def todf(self) -> pd.DataFrame: + return result_set_to_pandas(self) -def resultset_to_pandas(result_set: SessionDataSet) -> pd.DataFrame: +def result_set_to_pandas(result_set: SessionDataSet) -> pd.DataFrame: """ Transforms a SessionDataSet from IoTDB to a Pandas Data Frame Each Field from IoTDB is a column in Pandas :param result_set: :return: """ - return result_set.iotdb_rpc_data_set.resultset_to_pandas() + return result_set.iotdb_rpc_data_set.result_set_to_pandas() def get_typed_point(field: Field, none_value=None): choices = { # In Case of Boolean, cast to 0 / 1 - TSDataType.BOOLEAN: lambda field: 1 if field.get_bool_value() else 0, - TSDataType.TEXT: lambda field: field.get_string_value(), - TSDataType.FLOAT: lambda field: field.get_float_value(), - TSDataType.INT32: lambda field: field.get_int_value(), - TSDataType.DOUBLE: lambda field: field.get_double_value(), - TSDataType.INT64: lambda field: field.get_long_value(), + TSDataType.BOOLEAN: lambda f: 1 if f.get_bool_value() else 0, + TSDataType.TEXT: lambda f: f.get_string_value(), + TSDataType.FLOAT: lambda f: f.get_float_value(), + TSDataType.INT32: lambda f: f.get_int_value(), + TSDataType.DOUBLE: lambda f: f.get_double_value(), + TSDataType.INT64: lambda f: f.get_long_value(), + TSDataType.TIMESTAMP: lambda f: f.get_timestamp_value(), + TSDataType.STRING: lambda f: f.get_string_value(), + TSDataType.DATE: lambda f: f.get_date_value(), + TSDataType.BLOB: lambda f: f.get_binary_value(), } result_next_type: TSDataType = field.get_data_type() diff --git a/iotdb-client/client-py/iotdb/utils/Tablet.py b/iotdb-client/client-py/iotdb/utils/Tablet.py index 508361dc85748..9b241723fe5b7 100644 --- a/iotdb-client/client-py/iotdb/utils/Tablet.py +++ b/iotdb-client/client-py/iotdb/utils/Tablet.py @@ -17,26 +17,56 @@ # import struct +from enum import unique, IntEnum +from typing import List, Union +from iotdb.tsfile.utils.date_utils import parse_date_to_int from iotdb.utils.BitMap import BitMap +from iotdb.utils.IoTDBConstants import TSDataType + + +@unique +class ColumnType(IntEnum): + TAG = 0 + FIELD = 1 + ATTRIBUTE = 2 + + def n_copy(self, n): + result = [] + for i in range(n): + result.append(self) + return result class Tablet(object): - def __init__(self, device_id, measurements, data_types, values, timestamps): + def __init__( + self, + insert_target_name: str, + column_names: List[str], + data_types: List[TSDataType], + values: List[List], + timestamps: List[int], + column_types: List[ColumnType] = None, + ): """ creating a tablet for insertion - for example, considering device: root.sg1.d1 + for example using tree-model, considering device: root.sg1.d1 timestamps, m1, m2, m3 1, 125.3, True, text1 2, 111.6, False, text2 3, 688.6, True, text3 - Notice: From 0.13.0, the tablet can contain empty cell - The tablet will be sorted at the initialization by timestamps - :param device_id: String, IoTDB time series path to device layer (without sensor) - :param measurements: List, sensors - :param data_types: TSDataType List, specify value types for sensors + for example using table-model, considering table: table1 + timestamps, id1, attr1, m1 + 1, id:1, attr:1, 1.0 + 2, id:1, attr:1, 2.0 + 3, id:2, attr:2, 3.0 + Notice: The tablet will be sorted at the initialization by timestamps + :param insert_target_name: Str, DeviceId if using tree model or TableName when using table model. + :param column_names: Str List, names of columns + :param data_types: TSDataType List, specify value types for columns :param values: 2-D List, the values of each row should be the outer list element - :param timestamps: List, + :param timestamps: int List, contains the timestamps + :param column_types: ColumnType List, marking the type of each column, can be none for tree-view interfaces. """ if len(timestamps) != len(values): raise RuntimeError( @@ -51,11 +81,17 @@ def __init__(self, device_id, measurements, data_types, values, timestamps): self.__values = values self.__timestamps = timestamps - self.__device_id = device_id - self.__measurements = measurements + self.__insert_target_name = insert_target_name + self.__measurements = column_names self.__data_types = data_types self.__row_number = len(timestamps) - self.__column_number = len(measurements) + self.__column_number = len(column_names) + if column_types is None: + self.__column_types = ColumnType.n_copy( + ColumnType.FIELD, self.__column_number + ) + else: + self.__column_types = column_types @staticmethod def check_sorted(timestamps): @@ -70,11 +106,14 @@ def get_measurements(self): def get_data_types(self): return self.__data_types + def get_column_categories(self): + return self.__column_types + def get_row_number(self): return self.__row_number - def get_device_id(self): - return self.__device_id + def get_insert_target_name(self): + return self.__insert_target_name def get_binary_timestamps(self): format_str_list = [">"] @@ -89,13 +128,14 @@ def get_binary_timestamps(self): def get_binary_values(self): format_str_list = [">"] values_tobe_packed = [] - bitmaps = [] + bitmaps: List[Union[BitMap, None]] = [] has_none = False for i in range(self.__column_number): bitmap = None bitmaps.append(bitmap) - data_type_value = self.__data_types[i] - if data_type_value == 0: + data_type = self.__data_types[i] + # BOOLEAN + if data_type == 0: format_str_list.append(str(self.__row_number)) format_str_list.append("?") for j in range(self.__row_number): @@ -105,8 +145,8 @@ def get_binary_values(self): values_tobe_packed.append(False) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 1: + # INT32 + elif data_type == 1: format_str_list.append(str(self.__row_number)) format_str_list.append("i") for j in range(self.__row_number): @@ -116,8 +156,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 2: + # INT64 or TIMESTAMP + elif data_type == 2 or data_type == 8: format_str_list.append(str(self.__row_number)) format_str_list.append("q") for j in range(self.__row_number): @@ -127,8 +167,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 3: + # FLOAT + elif data_type == 3: format_str_list.append(str(self.__row_number)) format_str_list.append("f") for j in range(self.__row_number): @@ -138,8 +178,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 4: + # DOUBLE + elif data_type == 4: format_str_list.append(str(self.__row_number)) format_str_list.append("d") for j in range(self.__row_number): @@ -149,8 +189,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 5: + # TEXT, STRING, BLOB + elif data_type == 5 or data_type == 11 or data_type == 10: for j in range(self.__row_number): if self.__values[j][i] is not None: if isinstance(self.__values[j][i], str): @@ -171,7 +211,19 @@ def get_binary_values(self): values_tobe_packed.append(value_bytes) self.__mark_none_value(bitmaps, i, j) has_none = True - + # DATE + elif data_type == 9: + format_str_list.append(str(self.__row_number)) + format_str_list.append("i") + for j in range(self.__row_number): + if self.__values[j][i] is not None: + values_tobe_packed.append( + parse_date_to_int(self.__values[j][i]) + ) + else: + values_tobe_packed.append(0) + self.__mark_none_value(bitmaps, i, j) + has_none = True else: raise RuntimeError("Unsupported data type:" + str(self.__data_types[i])) diff --git a/iotdb-client/client-py/iotdb/utils/exception.py b/iotdb-client/client-py/iotdb/utils/exception.py new file mode 100644 index 0000000000000..2e233c6e7b02b --- /dev/null +++ b/iotdb-client/client-py/iotdb/utils/exception.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from iotdb.thrift.common.ttypes import TEndPoint, TSStatus + + +class IoTDBConnectionException(Exception): + def __init__(self, reason=None, cause=None, message=None): + if reason is not None: + super().__init__(reason) + elif cause is not None: + super().__init__(cause) + elif message is not None and cause is not None: + super().__init__(message, cause) + else: + super().__init__() + + +class StatementExecutionException(Exception): + def __init__(self, status: TSStatus = None, message=None): + if status is not None: + super().__init__(f"{status.code}: {status.message}") + elif message is not None: + super().__init__(message) + else: + super().__init__() + + +class RedirectException(Exception): + def __init__(self, redirect_info): + Exception.__init__(self) + if isinstance(redirect_info, TEndPoint): + self.redirect_node = redirect_info + else: + self.device_to_endpoint = redirect_info diff --git a/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py b/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py new file mode 100644 index 0000000000000..dc77136866938 --- /dev/null +++ b/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py @@ -0,0 +1,406 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# for package +import logging + +import numpy as np +import pandas as pd +from thrift.transport import TTransport + +from iotdb.thrift.rpc.IClientRPCService import TSFetchResultsReq, TSCloseOperationReq +from iotdb.tsfile.utils.date_utils import parse_int_to_date +from iotdb.tsfile.utils.tsblock_serde import deserialize +from iotdb.utils.exception import IoTDBConnectionException +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.rpc_utils import verify_success, convert_to_timestamp + +logger = logging.getLogger("IoTDB") +TIMESTAMP_STR = "Time" + + +class IoTDBRpcDataSet(object): + + def __init__( + self, + sql, + column_name_list, + column_type_list, + ignore_timestamp, + more_data, + query_id, + client, + statement_id, + session_id, + query_result, + fetch_size, + time_out, + zone_id, + time_precision, + column_index_2_tsblock_column_index_list, + ): + self.__statement_id = statement_id + self.__session_id = session_id + self.ignore_timestamp = ignore_timestamp + self.__sql = sql + self.__query_id = query_id + self.__client = client + self.__fetch_size = fetch_size + self.column_size = len(column_name_list) + self.__time_out = time_out + self.__more_data = more_data + + self.__column_name_list = [] + self.__column_type_list = [] + self.column_ordinal_dict = {} + self.column_name_2_tsblock_column_index_dict = {} + column_start_index = 1 + + start_index_for_column_index_2_tsblock_column_index_list = 0 + if not ignore_timestamp: + self.__column_name_list.append(TIMESTAMP_STR) + self.__column_type_list.append(TSDataType.INT64) + self.column_name_2_tsblock_column_index_dict[TIMESTAMP_STR] = -1 + self.column_ordinal_dict[TIMESTAMP_STR] = 1 + if column_index_2_tsblock_column_index_list is not None: + column_index_2_tsblock_column_index_list.insert(0, -1) + start_index_for_column_index_2_tsblock_column_index_list = 1 + column_start_index += 1 + + if column_index_2_tsblock_column_index_list is None: + column_index_2_tsblock_column_index_list = [] + if not ignore_timestamp: + start_index_for_column_index_2_tsblock_column_index_list = 1 + column_index_2_tsblock_column_index_list.append(-1) + for i in range(len(column_name_list)): + column_index_2_tsblock_column_index_list.append(i) + ts_block_column_size = ( + max(column_index_2_tsblock_column_index_list, default=0) + 1 + ) + self.__data_type_for_tsblock_column = [None] * ts_block_column_size + for i in range(len(column_name_list)): + name = column_name_list[i] + column_type = TSDataType[column_type_list[i]] + self.__column_name_list.append(name) + self.__column_type_list.append(column_type) + tsblock_column_index = column_index_2_tsblock_column_index_list[ + start_index_for_column_index_2_tsblock_column_index_list + i + ] + if tsblock_column_index != -1: + self.__data_type_for_tsblock_column[tsblock_column_index] = column_type + if name not in self.column_name_2_tsblock_column_index_dict: + self.column_ordinal_dict[name] = i + column_start_index + self.column_name_2_tsblock_column_index_dict[name] = ( + tsblock_column_index + ) + + self.__column_index_2_tsblock_column_index_list = ( + column_index_2_tsblock_column_index_list + ) + self.__query_result = query_result + self.__query_result_index = 0 + self.__is_closed = False + self.__empty_resultSet = False + self.has_cached_data_frame = False + self.data_frame = None + self.__zone_id = zone_id + self.__time_precision = time_precision + + def close(self): + if self.__is_closed: + return + if self.__client is not None: + try: + status = self.__client.closeOperation( + TSCloseOperationReq( + self.__session_id, self.__query_id, self.__statement_id + ) + ) + logger.debug( + "close session {}, message: {}".format( + self.__session_id, status.message + ) + ) + except TTransport.TException as e: + raise IoTDBConnectionException( + "close session {} failed because: ".format(self.__session_id), e + ) + + self.__is_closed = True + self.__client = None + + def next(self): + if not self.has_cached_data_frame: + self.construct_one_data_frame() + if self.has_cached_data_frame: + return True + if self.__empty_resultSet: + return False + if self.__more_data and self.fetch_results(): + self.construct_one_data_frame() + return True + return False + + def construct_one_data_frame(self): + if self.has_cached_data_frame or self.__query_result is None: + return + result = {} + has_pd_series = [] + for i in range(len(self.__column_index_2_tsblock_column_index_list)): + result[i] = [] + has_pd_series.append(False) + total_length = 0 + while self.__query_result_index < len(self.__query_result): + time_array, column_arrays, null_indicators, array_length = deserialize( + memoryview(self.__query_result[self.__query_result_index]) + ) + self.__query_result[self.__query_result_index] = None + self.__query_result_index += 1 + if self.ignore_timestamp is None or self.ignore_timestamp is False: + result[0].append(time_array) + total_length += array_length + for i, location in enumerate( + self.__column_index_2_tsblock_column_index_list + ): + if location < 0: + continue + data_type = self.__data_type_for_tsblock_column[location] + + column_array = column_arrays[location] + null_indicator = null_indicators[location] + if len(column_array) < array_length or ( + data_type == 0 and null_indicator is not None + ): + tmp_array = np.full(array_length, None, dtype=object) + if null_indicator is not None: + indexes = [not v for v in null_indicator] + if data_type == 0: + tmp_array[indexes] = column_array[indexes] + elif len(column_array) != 0: + tmp_array[indexes] = column_array + + # INT32, DATE + if data_type == 1 or data_type == 9: + tmp_array = pd.Series(tmp_array, dtype="Int32") + has_pd_series[i] = True + # INT64, TIMESTAMP + elif data_type == 2 or data_type == 8: + tmp_array = pd.Series(tmp_array, dtype="Int64") + has_pd_series[i] = True + # BOOLEAN + elif data_type == 0: + tmp_array = pd.Series(tmp_array, dtype="boolean") + has_pd_series[i] = True + # FLOAT, DOUBLE + elif data_type == 3 or data_type == 4: + tmp_array = pd.Series(tmp_array) + has_pd_series[i] = True + column_array = tmp_array + + result[i].append(column_array) + for k, v in result.items(): + if v is None or len(v) < 1 or v[0] is None: + result[k] = [] + elif not has_pd_series[k]: + res = np.empty(total_length, dtype=v[0].dtype) + np.concatenate(v, axis=0, out=res) + result[k] = res + else: + v = [x if isinstance(x, pd.Series) else pd.Series(x) for x in v] + result[k] = pd.concat(v, ignore_index=True) + + self.__query_result = None + self.data_frame = pd.DataFrame(result, dtype=object) + if not self.data_frame.empty: + self.has_cached_data_frame = True + + def has_cached_result(self): + return self.has_cached_data_frame + + def _has_next_result_set(self): + if (self.__query_result is not None) and ( + len(self.__query_result) > self.__query_result_index + ): + return True + if self.__empty_resultSet: + return False + if self.__more_data and self.fetch_results(): + return True + return False + + def result_set_to_pandas(self): + result = {} + for i in range(len(self.__column_index_2_tsblock_column_index_list)): + result[i] = [] + while self._has_next_result_set(): + time_array, column_arrays, null_indicators, array_length = deserialize( + memoryview(self.__query_result[self.__query_result_index]) + ) + self.__query_result[self.__query_result_index] = None + self.__query_result_index += 1 + if self.ignore_timestamp is None or self.ignore_timestamp is False: + if time_array.dtype.byteorder == ">" and len(time_array) > 0: + time_array = time_array.byteswap().view( + time_array.dtype.newbyteorder("<") + ) + result[0].append(time_array) + + for i, location in enumerate( + self.__column_index_2_tsblock_column_index_list + ): + if location < 0: + continue + data_type = self.__data_type_for_tsblock_column[location] + column_array = column_arrays[location] + # BOOLEAN, INT32, INT64, FLOAT, DOUBLE, BLOB + if data_type in (0, 1, 2, 3, 4, 10): + data_array = column_array + if ( + data_type != 10 + and len(data_array) > 0 + and data_array.dtype.byteorder == ">" + ): + data_array = data_array.byteswap().view( + data_array.dtype.newbyteorder("<") + ) + # TEXT, STRING + elif data_type in (5, 11): + data_array = np.array([x.decode("utf-8") for x in column_array]) + # TIMESTAMP + elif data_type == 8: + data_array = pd.Series( + [ + convert_to_timestamp( + x, self.__time_precision, self.__zone_id + ) + for x in column_array + ], + dtype=object, + ) + # DATE + elif data_type == 9: + data_array = pd.Series(column_array).apply(parse_int_to_date) + else: + raise RuntimeError("unsupported data type {}.".format(data_type)) + + null_indicator = null_indicators[location] + if len(data_array) < array_length or ( + data_type == 0 and null_indicator is not None + ): + tmp_array = [] + # BOOLEAN, INT32, INT64 + if data_type == 0 or data_type == 1 or data_type == 2: + tmp_array = np.full(array_length, pd.NA, dtype=object) + # FLOAT, DOUBLE + elif data_type == 3 or data_type == 4: + tmp_array = np.full( + array_length, np.nan, dtype=data_type.np_dtype() + ) + # TEXT, STRING, BLOB, DATE, TIMESTAMP + elif ( + data_type == 5 + or data_type == 11 + or data_type == 10 + or data_type == 9 + or data_type == 8 + ): + tmp_array = np.full(array_length, None, dtype=object) + + if null_indicator is not None: + indexes = [not v for v in null_indicator] + if data_type == 0: + tmp_array[indexes] = data_array[indexes] + elif len(data_array) != 0: + tmp_array[indexes] = data_array + + if data_type == 1: + tmp_array = pd.Series(tmp_array).astype("Int32") + elif data_type == 2: + tmp_array = pd.Series(tmp_array).astype("Int64") + elif data_type == 0: + tmp_array = pd.Series(tmp_array).astype("boolean") + + data_array = tmp_array + + result[i].append(data_array) + + for k, v in result.items(): + if v is None or len(v) < 1 or v[0] is None: + result[k] = [] + elif v[0].dtype == "Int32": + v = [x if isinstance(x, pd.Series) else pd.Series(x) for x in v] + result[k] = pd.concat(v, ignore_index=True).astype("Int32") + elif v[0].dtype == "Int64": + v = [x if isinstance(x, pd.Series) else pd.Series(x) for x in v] + result[k] = pd.concat(v, ignore_index=True).astype("Int64") + elif v[0].dtype == bool: + result[k] = pd.Series(np.concatenate(v, axis=0)).astype("boolean") + else: + result[k] = np.concatenate(v, axis=0) + + df = pd.DataFrame(result) + df.columns = self.__column_name_list + return df + + def fetch_results(self): + if self.__is_closed: + raise IoTDBConnectionException("This DataSet is already closed") + request = TSFetchResultsReq( + self.__session_id, + self.__sql, + self.__fetch_size, + self.__query_id, + True, + self.__time_out, + self.__statement_id, + ) + try: + resp = self.__client.fetchResultsV2(request) + verify_success(resp.status) + self.__more_data = resp.moreData + if not resp.hasResultSet: + self.__empty_resultSet = True + else: + self.__query_result = resp.queryResult + self.__query_result_index = 0 + return resp.hasResultSet + except TTransport.TException as e: + raise IoTDBConnectionException( + "Cannot fetch result from server, because of network connection: ", e + ) + + def find_column_name_by_index(self, column_index): + if column_index <= 0: + raise Exception("Column index should start from 1") + if column_index > len(self.__column_name_list): + raise Exception( + "column index {} out of range {}".format(column_index, self.column_size) + ) + return self.__column_name_list[column_index - 1] + + def get_fetch_size(self): + return self.__fetch_size + + def set_fetch_size(self, fetch_size): + self.__fetch_size = fetch_size + + def get_column_names(self): + return self.__column_name_list + + def get_column_types(self): + return self.__column_type_list diff --git a/iotdb-client/client-py/iotdb/utils/rpc_utils.py b/iotdb-client/client-py/iotdb/utils/rpc_utils.py new file mode 100644 index 0000000000000..7b292737f0c69 --- /dev/null +++ b/iotdb-client/client-py/iotdb/utils/rpc_utils.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import logging + +import pandas as pd +from pandas._libs import OutOfBoundsDatetime +from tzlocal import get_localzone_name + +from iotdb.thrift.common.ttypes import TSStatus +from iotdb.utils.exception import RedirectException, StatementExecutionException + +logger = logging.getLogger("IoTDB") + +SUCCESS_STATUS = 200 +MULTIPLE_ERROR = 302 +REDIRECTION_RECOMMEND = 400 + + +def verify_success(status: TSStatus): + """ + verify success of operation + :param status: execution result status + """ + if status.code == MULTIPLE_ERROR: + verify_success_by_list(status.subStatus) + return 0 + if status.code == SUCCESS_STATUS or status.code == REDIRECTION_RECOMMEND: + return 0 + + raise StatementExecutionException(status) + + +def verify_success_by_list(status_list: list): + """ + verify success of operation + :param status_list: execution result status + """ + error_messages = [ + status.message + for status in status_list + if status.code not in {SUCCESS_STATUS, REDIRECTION_RECOMMEND} + ] + if error_messages: + message = f"{MULTIPLE_ERROR}: {'; '.join(error_messages)}" + raise StatementExecutionException(message=message) + + +def verify_success_with_redirection(status: TSStatus): + verify_success(status) + if status.redirectNode is not None: + raise RedirectException(status.redirectNode) + return 0 + + +def verify_success_with_redirection_for_multi_devices(status: TSStatus, devices: list): + verify_success(status) + if status.code == MULTIPLE_ERROR or status.code == REDIRECTION_RECOMMEND: + device_to_endpoint = {} + for i in range(len(status.subStatus)): + if status.subStatus[i].redirectNode is not None: + device_to_endpoint[devices[i]] = status.subStatus[i].redirectNode + raise RedirectException(device_to_endpoint) + + +def convert_to_timestamp(time: int, precision: str, timezone: str): + try: + return pd.Timestamp(time, unit=precision, tz=timezone) + except OutOfBoundsDatetime: + return pd.Timestamp(time, unit=precision).tz_localize(timezone) + except ValueError: + logger.warning( + f"Timezone string '{timezone}' cannot be recognized by pandas. " + f"Falling back to local timezone: '{get_localzone_name()}'." + ) + return pd.Timestamp(time, unit=precision, tz=get_localzone_name()) + + +unit_map = { + "ms": "milliseconds", + "us": "microseconds", + "ns": "nanoseconds", +} + + +def isoformat(ts: pd.Timestamp, unit: str): + if unit not in unit_map: + raise ValueError(f"Unsupported unit: {unit}") + try: + return ts.isoformat(timespec=unit_map[unit]) + except ValueError: + logger.warning( + f"Timezone string '{unit_map[unit]}' cannot be recognized by old version pandas. " + f"Falling back to use auto timespec'." + ) + return ts.isoformat() diff --git a/iotdb-client/client-py/pom.xml b/iotdb-client/client-py/pom.xml index 0d942986ac8ab..5a644d5d1430d 100644 --- a/iotdb-client/client-py/pom.xml +++ b/iotdb-client/client-py/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-python-api IoTDB: Client: Python-API @@ -34,13 +34,19 @@ org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT provided org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-thrift-confignode + 2.0.4-SNAPSHOT provided @@ -114,13 +120,13 @@ ${basedir}/../../iotdb-protocol/thrift-datanode/target/generated-sources-python/iotdb/thrift/ - - **/rpc/* - ${basedir}/../../iotdb-protocol/thrift-commons/target/generated-sources-python/iotdb/thrift/ + + ${basedir}/../../iotdb-protocol/thrift-confignode/target/generated-sources-python/iotdb/thrift/ + @@ -151,7 +157,7 @@ copy-resources - generate-sources + process-resources utf-8 ${basedir}/ @@ -195,6 +201,7 @@ org.apache.iotdb:iotdb-thrift-commons org.apache.iotdb:iotdb-thrift + org.apache.iotdb:iotdb-thrift-confignode diff --git a/iotdb-client/client-py/release.sh b/iotdb-client/client-py/release.sh index 692047b8b57b6..60e466eea7e7c 100755 --- a/iotdb-client/client-py/release.sh +++ b/iotdb-client/client-py/release.sh @@ -25,6 +25,7 @@ python3 --version rm -Rf build rm -Rf dist rm -Rf iotdb_session.egg_info +rm -f pyproject.toml # (Re-)build generated code (cd ../..; mvn clean package -pl iotdb-client/client-py -am) diff --git a/iotdb-client/client-py/requirements.txt b/iotdb-client/client-py/requirements.txt index 490393d157ce2..980a7442a544f 100644 --- a/iotdb-client/client-py/requirements.txt +++ b/iotdb-client/client-py/requirements.txt @@ -21,5 +21,6 @@ pandas>=1.0.0 numpy>=1.0.0 thrift>=0.14.1 # SQLAlchemy Dialect -sqlalchemy<1.5,>=1.4 +sqlalchemy>=1.4 sqlalchemy-utils>=0.37.8 +tzlocal>=4.0 diff --git a/iotdb-client/client-py/resources/pyproject.toml b/iotdb-client/client-py/resources/pyproject.toml index 09a94636e64fb..2b32eefd59b9b 100644 --- a/iotdb-client/client-py/resources/pyproject.toml +++ b/iotdb-client/client-py/resources/pyproject.toml @@ -28,11 +28,10 @@ readme = "README.md" authors = [ { name = "Apache IoTDB", email = "dev@iotdb.apache.org" } ] -license = { text = "Apache License, Version 2.0"} +license = "Apache-2.0" keywords = ["iotdb", "apache", "client", "API"] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules" @@ -42,8 +41,9 @@ dependencies = [ "thrift>=0.14.1", "pandas>=1.0.0", "numpy>=1.0.0", - "sqlalchemy<1.5,>=1.4", - "sqlalchemy-utils>=0.37.8" + "sqlalchemy>=1.4", + "sqlalchemy-utils>=0.37.8", + "tzlocal>=4.0" ] [project.urls] diff --git a/iotdb-client/client-py/session_aligned_timeseries_example.py b/iotdb-client/client-py/session_aligned_timeseries_example.py new file mode 100644 index 0000000000000..450d69f281883 --- /dev/null +++ b/iotdb-client/client-py/session_aligned_timeseries_example.py @@ -0,0 +1,227 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Uncomment the following line to use apache-iotdb module installed by pip3 + +from iotdb.Session import Session +from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor +from iotdb.utils.Tablet import Tablet + +# creating session connection. +ip = "127.0.0.1" +port_ = "6667" +username_ = "root" +password_ = "root" +session = Session( + ip, port_, username_, password_, fetch_size=1024, zone_id="Asia/Shanghai" +) +session.open(False) + +# set and delete databases +session.set_storage_group("root.sg_test_01") +session.set_storage_group("root.sg_test_02") +session.set_storage_group("root.sg_test_03") +session.set_storage_group("root.sg_test_04") +session.delete_storage_group("root.sg_test_02") +session.delete_storage_groups(["root.sg_test_03", "root.sg_test_04"]) + +# setting aligned time series. +measurements_lst_ = [ + "s_01", + "s_02", + "s_03", +] +data_type_lst_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, +] +encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] +compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] +session.create_aligned_time_series( + "root.sg_test_01.d_02", + measurements_lst_, + data_type_lst_, + encoding_lst_, + compressor_lst_, +) + +# setting more aligned time series once. +measurements_lst_ = [ + "s_04", + "s_05", + "s_06", + "s_07", + "s_08", + "s_09", +] +data_type_lst_ = [ + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, +] +encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] +compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] +session.create_aligned_time_series( + "root.sg_test_01.d_02", + measurements_lst_, + data_type_lst_, + encoding_lst_, + compressor_lst_, +) + +# delete time series +session.delete_time_series( + [ + "root.sg_test_01.d_02.s_07", + "root.sg_test_01.d_02.s_08", + "root.sg_test_01.d_02.s_09", + ] +) + +# checking time series +print( + "s_07 expecting False, checking result: ", + session.check_time_series_exists("root.sg_test_01.d_02.s_07"), +) +print( + "s_03 expecting True, checking result: ", + session.check_time_series_exists("root.sg_test_01.d_02.s_03"), +) + +# insert one aligned record into the database. +measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] +values_ = [False, 10, 11, 1.1, 10011.1, "test_record"] +data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, +] +session.insert_aligned_record( + "root.sg_test_01.d_02", 1, measurements_, data_types_, values_ +) + +# insert multiple aligned records into database +measurements_list_ = [ + ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], + ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], +] +values_list_ = [ + [False, 22, 33, 4.4, 55.1, "test_records01"], + [True, 77, 88, 1.25, 8.125, "test_records02"], +] +data_type_list_ = [data_types_, data_types_] +device_ids_ = ["root.sg_test_01.d_02", "root.sg_test_01.d_02"] +session.insert_aligned_records( + device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ +) + +# insert one aligned tablet into the database. +values_ = [ + [False, 10, 11, 1.1, 10011.1, "test01"], + [True, 100, 11111, 1.25, 101.0, "test02"], + [False, 100, 1, 188.1, 688.25, "test03"], + [True, 0, 0, 0, 6.25, "test04"], +] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. +timestamps_ = [4, 5, 6, 7] +tablet_ = Tablet( + "root.sg_test_01.d_02", measurements_, data_types_, values_, timestamps_ +) +session.insert_aligned_tablet(tablet_) + +# insert multiple aligned tablets into database +tablet_01 = Tablet( + "root.sg_test_01.d_02", measurements_, data_types_, values_, [8, 9, 10, 11] +) +tablet_02 = Tablet( + "root.sg_test_01.d_02", measurements_, data_types_, values_, [12, 13, 14, 15] +) +session.insert_aligned_tablets([tablet_01, tablet_02]) + +# insert one aligned tablet with empty cells into the database. +values_ = [ + [None, 10, 11, 1.1, 10011.1, "test01"], + [True, None, 11111, 1.25, 101.0, "test02"], + [False, 100, 1, None, 688.25, "test03"], + [True, 0, 0, 0, 6.25, None], +] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. +timestamps_ = [16, 17, 18, 19] +tablet_ = Tablet( + "root.sg_test_01.d_02", measurements_, data_types_, values_, timestamps_ +) +session.insert_aligned_tablet(tablet_) + +# insert aligned records of one device +time_list = [1, 2, 3] +measurements_list = [ + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], +] +data_types_list = [ + [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], + [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], + [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], +] +values_list = [[False, 22, 33], [True, 1, 23], [False, 15, 26]] + +session.insert_aligned_records_of_one_device( + "root.sg_test_01.d_02", time_list, measurements_list, data_types_list, values_list +) + +# execute non-query sql statement +session.execute_non_query_statement( + "insert into root.sg_test_01.d_02(timestamp, s_02) aligned values(16, 188)" +) + +# execute sql query statement +with session.execute_query_statement( + "select * from root.sg_test_01.d_02" +) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + +# insert aligned string record into the database. +time_list = [1, 2, 3] +measurements_list = [ + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], +] +values_list = [["False", "22", "33"], ["True", "1", "23"], ["False", "15", "26"]] +session.insert_aligned_string_records_of_one_device( + "root.sg_test_01.d_04", + time_list, + measurements_list, + values_list, +) + +# delete database +session.delete_storage_group("root.sg_test_01") + +# close session connection. +session.close() + +print("All executions done!!") diff --git a/iotdb-client/client-py/session_example.py b/iotdb-client/client-py/session_example.py new file mode 100644 index 0000000000000..d0a6a3aba8e37 --- /dev/null +++ b/iotdb-client/client-py/session_example.py @@ -0,0 +1,420 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Uncomment the following line to use apache-iotdb module installed by pip3 +import numpy as np +from datetime import date +from iotdb.Session import Session +from iotdb.utils.BitMap import BitMap +from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor +from iotdb.utils.Tablet import Tablet +from iotdb.utils.NumpyTablet import NumpyTablet + +# creating session connection. +ip = "127.0.0.1" +port_ = "6667" +username_ = "root" +password_ = "root" +# session = Session(ip, port_, username_, password_, fetch_size=1024, zone_id="Asia/Shanghai", enable_redirection=True) +session = Session.init_from_node_urls( + node_urls=["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"], + user="root", + password="root", + fetch_size=1024, + zone_id="Asia/Shanghai", + enable_redirection=True, +) +session.open(False) + +# create and delete databases +session.set_storage_group("root.sg_test_01") +session.set_storage_group("root.sg_test_02") +session.set_storage_group("root.sg_test_03") +session.set_storage_group("root.sg_test_04") +session.delete_storage_group("root.sg_test_02") +session.delete_storage_groups(["root.sg_test_03", "root.sg_test_04"]) + +# setting time series. +session.create_time_series( + "root.sg_test_01.d_01.s_01", TSDataType.BOOLEAN, TSEncoding.PLAIN, Compressor.SNAPPY +) +session.create_time_series( + "root.sg_test_01.d_01.s_02", TSDataType.INT32, TSEncoding.PLAIN, Compressor.SNAPPY +) +session.create_time_series( + "root.sg_test_01.d_01.s_03", TSDataType.INT64, TSEncoding.PLAIN, Compressor.SNAPPY +) +session.create_time_series( + "root.sg_test_01.d_02.s_01", + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + Compressor.SNAPPY, + None, + {"tag1": "v1"}, + {"description": "v1"}, + "temperature", +) + +# setting multiple time series once. +ts_path_lst_ = [ + "root.sg_test_01.d_01.s_04", + "root.sg_test_01.d_01.s_05", + "root.sg_test_01.d_01.s_06", + "root.sg_test_01.d_01.s_07", + "root.sg_test_01.d_01.s_08", + "root.sg_test_01.d_01.s_09", +] +data_type_lst_ = [ + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, +] +encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] +compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] +session.create_multi_time_series( + ts_path_lst_, data_type_lst_, encoding_lst_, compressor_lst_ +) + +ts_path_lst_ = [ + "root.sg_test_01.d_02.s_04", + "root.sg_test_01.d_02.s_05", + "root.sg_test_01.d_02.s_06", + "root.sg_test_01.d_02.s_07", + "root.sg_test_01.d_02.s_08", + "root.sg_test_01.d_02.s_09", +] +data_type_lst_ = [ + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, +] +encoding_lst_ = [TSEncoding.PLAIN for _ in range(len(data_type_lst_))] +compressor_lst_ = [Compressor.SNAPPY for _ in range(len(data_type_lst_))] +tags_lst_ = [{"tag2": "v2"} for _ in range(len(data_type_lst_))] +attributes_lst_ = [{"description": "v2"} for _ in range(len(data_type_lst_))] +session.create_multi_time_series( + ts_path_lst_, + data_type_lst_, + encoding_lst_, + compressor_lst_, + None, + tags_lst_, + attributes_lst_, + None, +) + +# delete time series +session.delete_time_series( + [ + "root.sg_test_01.d_01.s_07", + "root.sg_test_01.d_01.s_08", + "root.sg_test_01.d_01.s_09", + ] +) + +# checking time series +print( + "s_07 expecting False, checking result: ", + session.check_time_series_exists("root.sg_test_01.d_01.s_07"), +) +print( + "s_03 expecting True, checking result: ", + session.check_time_series_exists("root.sg_test_01.d_01.s_03"), +) +print( + "d_02.s_01 expecting True, checking result: ", + session.check_time_series_exists("root.sg_test_01.d_02.s_01"), +) +print( + "d_02.s_06 expecting True, checking result: ", + session.check_time_series_exists("root.sg_test_01.d_02.s_06"), +) + +# insert one record into the database. +measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] +values_ = [False, 10, 11, 1.1, 10011.1, "test_record"] +data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, +] +session.insert_record("root.sg_test_01.d_01", 1, measurements_, data_types_, values_) + +# insert multiple records into database +measurements_list_ = [ + ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], + ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], +] +values_list_ = [ + [False, 22, 33, 4.4, 55.1, "test_records01"], + [True, 77, 88, 1.25, 8.125, bytes("test_records02", "utf-8")], +] +data_type_list_ = [data_types_, data_types_] +device_ids_ = ["root.sg_test_01.d_01", "root.sg_test_01.d_01"] +session.insert_records( + device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ +) + +# insert one tablet into the database. +values_ = [ + [False, 10, 11, 1.1, 10011.1, "test01"], + [True, 100, 11111, 1.25, 101.0, "test02"], + [False, 100, 1, 188.1, 688.25, "test03"], + [True, 0, 0, 0, 6.25, "test04"], +] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. +timestamps_ = [4, 5, 6, 7] +tablet_ = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ +) +session.insert_tablet(tablet_) + +# insert one numpy tablet into the database. +np_values_ = [ + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), +] +np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) +np_tablet_ = NumpyTablet( + "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ +) +session.insert_tablet(np_tablet_) + +# insert one unsorted numpy tablet into the database. +np_values_unsorted = [ + np.array([False, False, False, True, True], np.dtype(">?")), + np.array([0, 10, 100, 1000, 10000], np.dtype(">i4")), + np.array([1, 11, 111, 1111, 11111], np.dtype(">i8")), + np.array([1.1, 1.25, 188.1, 0, 8.999], np.dtype(">f4")), + np.array([10011.1, 101.0, 688.25, 6.25, 8, 776], np.dtype(">f8")), + np.array(["test09", "test08", "test07", "test06", "test05"]), +] +np_timestamps_unsorted = np.array([9, 8, 7, 6, 5], np.dtype(">i8")) +np_tablet_unsorted = NumpyTablet( + "root.sg_test_01.d_02", + measurements_, + data_types_, + np_values_unsorted, + np_timestamps_unsorted, +) + +# insert one numpy tablet into the database. +np_values_ = [ + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"]), +] +np_timestamps_ = np.array([98, 99, 100, 101], TSDataType.INT64.np_dtype()) +np_bitmaps_ = [] +for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) +np_bitmaps_[0].mark(0) +np_bitmaps_[1].mark(1) +np_bitmaps_[2].mark(2) +np_bitmaps_[4].mark(3) +np_bitmaps_[5].mark(3) +np_tablet_with_none = NumpyTablet( + "root.sg_test_01.d_02", + measurements_, + data_types_, + np_values_, + np_timestamps_, + np_bitmaps_, +) +session.insert_tablet(np_tablet_with_none) + + +session.insert_tablet(np_tablet_unsorted) +print(np_tablet_unsorted.get_timestamps()) +for value in np_tablet_unsorted.get_values(): + print(value) + +# insert multiple tablets into database +tablet_01 = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, [8, 9, 10, 11] +) +tablet_02 = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, [12, 13, 14, 15] +) +session.insert_tablets([tablet_01, tablet_02]) + +# insert one tablet with empty cells into the database. +values_ = [ + [None, 10, 11, 1.1, 10011.1, "test01"], + [True, None, 11111, 1.25, 101.0, "test02"], + [False, 100, 1, None, 688.25, "test03"], + [True, 0, 0, 0, 6.25, None], +] # Non-ASCII text will cause error since bytes can only hold 0-128 nums. +timestamps_ = [16, 17, 18, 19] +tablet_ = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ +) +session.insert_tablet(tablet_) + +# insert records of one device +time_list = [1, 2, 3] +measurements_list = [ + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], +] +data_types_list = [ + [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], + [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], + [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64], +] +values_list = [[False, 22, 33], [True, 1, 23], [False, 15, 26]] + +session.insert_records_of_one_device( + "root.sg_test_01.d_01", time_list, measurements_list, data_types_list, values_list +) + +# execute non-query sql statement +session.execute_non_query_statement( + "insert into root.sg_test_01.d_01(timestamp, s_02) values(16, 188)" +) + +# execute sql query statement +with session.execute_query_statement( + "select * from root.sg_test_01.d_01" +) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) +# execute sql query statement +with session.execute_query_statement( + "select s_01, s_02, s_03, s_04, s_05, s_06 from root.sg_test_01.d_02" +) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + +# execute statement +with session.execute_statement( + "select * from root.sg_test_01.d_01" +) as session_data_set: + while session_data_set.has_next(): + print(session_data_set.next()) + +session.execute_statement( + "insert into root.sg_test_01.d_01(timestamp, s_02) values(16, 188)" +) + +# insert string records of one device +time_list = [1, 2, 3] +measurements_list = [ + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], + ["s_01", "s_02", "s_03"], +] +values_list = [["False", "22", "33"], ["True", "1", "23"], ["False", "15", "26"]] + +session.insert_string_records_of_one_device( + "root.sg_test_01.d_03", + time_list, + measurements_list, + values_list, +) + +with session.execute_raw_data_query( + ["root.sg_test_01.d_03.s_01", "root.sg_test_01.d_03.s_02"], 1, 4 +) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + +with session.execute_last_data_query( + ["root.sg_test_01.d_03.s_01", "root.sg_test_01.d_03.s_02"], 0 +) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + +# insert tablet with new data types +measurements_new_type = ["s_01", "s_02", "s_03", "s_04"] +data_types_new_type = [ + TSDataType.DATE, + TSDataType.TIMESTAMP, + TSDataType.BLOB, + TSDataType.STRING, +] +values_new_type = [ + [date(2024, 1, 1), 1, b"\x12\x34", "test01"], + [date(2024, 1, 2), 2, b"\x12\x34", "test02"], + [date(2024, 1, 3), 3, b"\x12\x34", "test03"], + [date(2024, 1, 4), 4, b"\x12\x34", "test04"], +] +timestamps_new_type = [1, 2, 3, 4] +tablet_new_type = Tablet( + "root.sg_test_01.d_04", + measurements_new_type, + data_types_new_type, + values_new_type, + timestamps_new_type, +) +session.insert_tablet(tablet_new_type) +np_values_new_type = [ + np.array([date(2024, 2, 4), date(2024, 3, 4), date(2024, 4, 4), date(2024, 5, 4)]), + np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()), + np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]), + np.array(["test01", "test02", "test03", "test04"]), +] +np_timestamps_new_type = np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()) +np_tablet_new_type = NumpyTablet( + "root.sg_test_01.d_04", + measurements_new_type, + data_types_new_type, + np_values_new_type, + np_timestamps_new_type, +) +session.insert_tablet(np_tablet_new_type) +with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" +) as dataset: + print(dataset.get_column_names()) + while dataset.has_next(): + print(dataset.next()) + +with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" +) as dataset: + df = dataset.todf() + print(df.to_string()) + +# delete database +session.delete_storage_group("root.sg_test_01") + +# close session connection. +session.close() + +print("All executions done!!") diff --git a/iotdb-client/client-py/session_pool_example.py b/iotdb-client/client-py/session_pool_example.py new file mode 100644 index 0000000000000..64a754087fae5 --- /dev/null +++ b/iotdb-client/client-py/session_pool_example.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import threading + +from iotdb.SessionPool import PoolConfig, SessionPool +from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor + +STORAGE_GROUP_NAME = "root.test" + + +def prepare_data(): + print("create database") + # Get a session from the pool + session = session_pool.get_session() + + session.set_storage_group(STORAGE_GROUP_NAME) + session.create_time_series( + "root.test.d0.s0", TSDataType.BOOLEAN, TSEncoding.PLAIN, Compressor.SNAPPY + ) + session.create_time_series( + "root.test.d0.s1", TSDataType.INT32, TSEncoding.PLAIN, Compressor.SNAPPY + ) + session.create_time_series( + "root.test.d0.s2", TSDataType.INT64, TSEncoding.PLAIN, Compressor.SNAPPY + ) + + # create multi time series + # setting multiple time series once. + ts_path_lst = [ + "root.test.d1.s0", + "root.test.d1.s1", + "root.test.d1.s2", + "root.test.d1.s3", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + ] + encoding_lst = [TSEncoding.PLAIN for _ in range(len(data_type_lst))] + compressor_lst = [Compressor.SNAPPY for _ in range(len(data_type_lst))] + session.create_multi_time_series( + ts_path_lst, data_type_lst, encoding_lst, compressor_lst + ) + + # delete time series root.test.d1.s3 + session.delete_time_series(["root.test.d1.s3"]) + + print("now the timeseries are:") + # show result + res = session.execute_query_statement("show timeseries root.test.**") + while res.has_next(): + print(res.next()) + + session_pool.put_back(session) + + +def insert_data(num: int): + print("insert data for root.test.d" + str(num)) + device = ["root.test.d" + str(num)] + measurements = ["s0", "s1", "s2"] + values = [False, 10, 11] + data_types = [TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.INT64] + # Get a session from the pool + session = session_pool.get_session() + session.insert_records(device, [1], [measurements], [data_types], [values]) + + session_pool.put_back(session) + + +def query_data(): + # Get a session from the pool + session = session_pool.get_session() + + print("get data from root.test.d0") + res = session.execute_query_statement("select * from root.test.d0") + while res.has_next(): + print(res.next()) + + print("get data from root.test.d1") + res = session.execute_query_statement("select * from root.test.d1") + while res.has_next(): + print(res.next()) + + session_pool.put_back(session) + + +def delete_data(): + session = session_pool.get_session() + session.delete_storage_group(STORAGE_GROUP_NAME) + print("data has been deleted. now the devices are:") + res = session.execute_statement("show devices root.test.**") + while res.has_next(): + print(res.next()) + session_pool.put_back(session) + + +ip = "127.0.0.1" +port = "6667" +username = "root" +password = "root" +pool_config = PoolConfig( + node_urls=["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"], + user_name=username, + password=password, + fetch_size=1024, + time_zone="Asia/Shanghai", + max_retry=3, +) +max_pool_size = 5 +wait_timeout_in_ms = 3000 + +# Create a session pool +session_pool = SessionPool(pool_config, max_pool_size, wait_timeout_in_ms) + +prepare_data() + +insert_thread1 = threading.Thread(target=insert_data, args=(0,)) +insert_thread2 = threading.Thread(target=insert_data, args=(1,)) + +insert_thread1.start() +insert_thread2.start() + +insert_thread1.join() +insert_thread2.join() + +query_data() +delete_data() +session_pool.close() +print("example is finished!") diff --git a/iotdb-client/client-py/session_ssl_example.py b/iotdb-client/client-py/session_ssl_example.py new file mode 100644 index 0000000000000..2d5a557afc449 --- /dev/null +++ b/iotdb-client/client-py/session_ssl_example.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from iotdb.SessionPool import PoolConfig, SessionPool +from iotdb.Session import Session +from iotdb.table_session import TableSession, TableSessionConfig + +ip = "127.0.0.1" +port_ = "6667" +username_ = "root" +password_ = "root" +# Configure SSL enabled +use_ssl = True +# Configure certificate path +ca_certs = "/path/server.crt" + + +def get_data(): + session = Session( + ip, port_, username_, password_, use_ssl=use_ssl, ca_certs=ca_certs + ) + session.open(False) + result = session.execute_query_statement("select * from root.eg.etth") + df = result.todf() + df.rename(columns={"Time": "date"}, inplace=True) + session.close() + return df + + +def get_data2(): + pool_config = PoolConfig( + host=ip, + port=port_, + user_name=username_, + password=password_, + fetch_size=1024, + time_zone="Asia/Shanghai", + max_retry=3, + use_ssl=use_ssl, + ca_certs=ca_certs, + ) + max_pool_size = 5 + wait_timeout_in_ms = 3000 + session_pool = SessionPool(pool_config, max_pool_size, wait_timeout_in_ms) + session = session_pool.get_session() + result = session.execute_query_statement("select * from root.eg.etth") + df = result.todf() + df.rename(columns={"Time": "date"}, inplace=True) + session_pool.put_back(session) + session_pool.close() + + +def get_table_data(): + pool_config = TableSessionConfig( + node_urls=["127.0.0.1:6667"], + username=username_, + password=password_, + fetch_size=1024, + time_zone="Asia/Shanghai", + use_ssl=use_ssl, + ca_certs=ca_certs, + ) + session = TableSession(pool_config) + result = session.execute_query_statement("select * from test") + df = result.todf() + session.close() + + +if __name__ == "__main__": + df = get_data() diff --git a/iotdb-client/client-py/table_model_session_example.py b/iotdb-client/client-py/table_model_session_example.py new file mode 100644 index 0000000000000..c9aa62b97a0ca --- /dev/null +++ b/iotdb-client/client-py/table_model_session_example.py @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import numpy as np + +from iotdb.table_session import TableSession, TableSessionConfig +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.Tablet import ColumnType, Tablet + +# creating session connection. +# don't specify database in constructor +config = TableSessionConfig( + node_urls=["localhost:6667"], + username="root", + password="root", + time_zone="Asia/Shanghai", +) +session = TableSession(config) + +session.execute_non_query_statement("CREATE DATABASE test1") +session.execute_non_query_statement("CREATE DATABASE test2") +session.execute_non_query_statement("use test2") + +# or use full qualified table name +session.execute_non_query_statement( + "create table test1.table1(" + "region_id STRING TAG, plant_id STRING TAG, device_id STRING TAG, " + "model STRING ATTRIBUTE, temperature FLOAT FIELD, humidity DOUBLE FIELD) with (TTL=3600000)" +) +session.execute_non_query_statement( + "create table table2(" + "region_id STRING TAG, plant_id STRING TAG, color STRING ATTRIBUTE, temperature FLOAT FIELD," + " speed DOUBLE FIELD) with (TTL=6600000)" +) + +# show tables from current database +with session.execute_query_statement("SHOW TABLES") as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + +# show tables by specifying another database +# using SHOW tables FROM +with session.execute_query_statement("SHOW TABLES FROM test1") as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + +session.close() + +# specify database in constructor +config = TableSessionConfig( + node_urls=["localhost:6667"], + username="root", + password="root", + database="test1", + time_zone="Asia/Shanghai", +) +session = TableSession(config) + +# show tables from current database +with session.execute_query_statement("SHOW TABLES") as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + +# change database to test2 +session.execute_non_query_statement("use test2") + +# show tables by specifying another database +# using SHOW tables FROM +with session.execute_query_statement("SHOW TABLES") as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + +session.close() + +# insert data by tablet +config = TableSessionConfig( + node_urls=["localhost:6667"], + username="root", + password="root", +) +session = TableSession(config) +session.execute_non_query_statement("CREATE DATABASE IF NOT EXISTS db1") +session.execute_non_query_statement('USE "db1"') +session.execute_non_query_statement( + "CREATE TABLE table5 (id1 string TAG, attr1 string attribute, " + + "m1 double " + + "FIELD)" +) + +column_names = [ + "id1", + "attr1", + "m1", +] +data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.DOUBLE, +] +column_types = [ColumnType.TAG, ColumnType.ATTRIBUTE, ColumnType.FIELD] +timestamps = [] +values = [] +for row in range(15): + timestamps.append(row) + values.append(["id:" + str(row), "attr:" + str(row), row * 1.0]) +tablet = Tablet("table5", column_names, data_types, values, timestamps, column_types) +session.insert(tablet) + +session.execute_non_query_statement("FLush") + +np_timestamps = np.arange(15, 30, dtype=np.dtype(">i8")) +np_values = [ + np.array(["id:{}".format(i) for i in range(15, 30)]), + np.array(["attr:{}".format(i) for i in range(15, 30)]), + np.linspace(15.0, 29.0, num=15, dtype=TSDataType.DOUBLE.np_dtype()), +] + +np_tablet = NumpyTablet( + "table5", + column_names, + data_types, + np_values, + np_timestamps, + column_types=column_types, +) +session.insert(np_tablet) + +with session.execute_query_statement("select * from table5 order by time") as dataset: + print(dataset.get_column_names()) + while dataset.has_next(): + row_record = dataset.next() + print(row_record.get_fields()[0].get_long_value()) + print(row_record.get_fields()[1].get_string_value()) + print(row_record.get_fields()[2].get_string_value()) + print(row_record.get_fields()[3].get_double_value()) + print(row_record) + +with session.execute_query_statement("select * from table5 order by time") as dataset: + df = dataset.todf() + print(df) + +# close session connection. +session.close() diff --git a/iotdb-client/client-py/table_model_session_pool_example.py b/iotdb-client/client-py/table_model_session_pool_example.py new file mode 100644 index 0000000000000..bd190a8b47328 --- /dev/null +++ b/iotdb-client/client-py/table_model_session_pool_example.py @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import threading + +import numpy as np + +from iotdb.table_session_pool import TableSessionPool, TableSessionPoolConfig +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.Tablet import ColumnType, Tablet + + +def prepare_data(): + print("create database") + # Get a session from the pool + session = session_pool.get_session() + session.execute_non_query_statement("CREATE DATABASE IF NOT EXISTS db1") + session.execute_non_query_statement('USE "db1"') + session.execute_non_query_statement( + "CREATE TABLE table0 (id1 string tag, attr1 string attribute, " + + "m1 double " + + "field)" + ) + session.execute_non_query_statement( + "CREATE TABLE table1 (id1 string tag, attr1 string attribute, " + + "m1 double " + + "field)" + ) + + print("now the tables are:") + # show result + res = session.execute_query_statement("SHOW TABLES") + while res.has_next(): + print(res.next()) + + session.close() + + +def insert_data(num: int): + print("insert data for table" + str(num)) + # Get a session from the pool + session = session_pool.get_session() + column_names = [ + "id1", + "attr1", + "m1", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.DOUBLE, + ] + column_types = [ColumnType.TAG, ColumnType.ATTRIBUTE, ColumnType.FIELD] + timestamps = [] + values = [] + for row in range(15): + timestamps.append(row) + values.append(["id:" + str(row), "attr:" + str(row), row * 1.0]) + tablet = Tablet( + "table" + str(num), column_names, data_types, values, timestamps, column_types + ) + session.insert(tablet) + session.execute_non_query_statement("FLush") + + np_timestamps = np.arange(15, 30, dtype=np.dtype(">i8")) + np_values = [ + np.array(["id:{}".format(i) for i in range(15, 30)]), + np.array(["attr:{}".format(i) for i in range(15, 30)]), + np.linspace(15.0, 29.0, num=15, dtype=TSDataType.DOUBLE.np_dtype()), + ] + + np_tablet = NumpyTablet( + "table" + str(num), + column_names, + data_types, + np_values, + np_timestamps, + column_types=column_types, + ) + session.insert(np_tablet) + session.close() + + +def query_data(): + # Get a session from the pool + session = session_pool.get_session() + + print("get data from table0") + res = session.execute_query_statement("select * from table0") + while res.has_next(): + print(res.next()) + + print("get data from table1") + res = session.execute_query_statement("select * from table0") + while res.has_next(): + print(res.next()) + + session.close() + + +def delete_data(): + session = session_pool.get_session() + session.execute_non_query_statement("drop database db1") + print("data has been deleted. now the databases are:") + res = session.execute_query_statement("show databases") + while res.has_next(): + print(res.next()) + session.close() + + +# Create a session pool +username = "root" +password = "root" +node_urls = ["127.0.0.1:6667", "127.0.0.1:6668", "127.0.0.1:6669"] +fetch_size = 1024 +database = "db1" +max_pool_size = 5 +wait_timeout_in_ms = 3000 +config = TableSessionPoolConfig( + node_urls=node_urls, + username=username, + password=password, + database=database, + max_pool_size=max_pool_size, + fetch_size=fetch_size, + wait_timeout_in_ms=wait_timeout_in_ms, +) +session_pool = TableSessionPool(config) + +prepare_data() + +insert_thread1 = threading.Thread(target=insert_data, args=(0,)) +insert_thread2 = threading.Thread(target=insert_data, args=(1,)) + +insert_thread1.start() +insert_thread2.start() + +insert_thread1.join() +insert_thread2.join() + +query_data() +delete_data() +session_pool.close() +print("example is finished!") diff --git a/iotdb-client/client-py/iotdb/dbapi/tests/__init__.py b/iotdb-client/client-py/tests/integration/dbapi/__init__.py similarity index 100% rename from iotdb-client/client-py/iotdb/dbapi/tests/__init__.py rename to iotdb-client/client-py/tests/integration/dbapi/__init__.py diff --git a/iotdb-client/client-py/iotdb/dbapi/tests/test_connection.py b/iotdb-client/client-py/tests/integration/dbapi/test_connection.py similarity index 96% rename from iotdb-client/client-py/iotdb/dbapi/tests/test_connection.py rename to iotdb-client/client-py/tests/integration/dbapi/test_connection.py index d0d878ce2be07..590fe5f7da5bf 100644 --- a/iotdb-client/client-py/iotdb/dbapi/tests/test_connection.py +++ b/iotdb-client/client-py/tests/integration/dbapi/test_connection.py @@ -16,7 +16,7 @@ # under the License. # -from iotdb.IoTDBContainer import IoTDBContainer +from tests.integration.iotdb_container import IoTDBContainer from iotdb.dbapi import connect final_flag = True diff --git a/iotdb-client/client-py/iotdb/dbapi/tests/test_cursor.py b/iotdb-client/client-py/tests/integration/dbapi/test_cursor.py similarity index 98% rename from iotdb-client/client-py/iotdb/dbapi/tests/test_cursor.py rename to iotdb-client/client-py/tests/integration/dbapi/test_cursor.py index 4737aa95eafbb..eb6f70b838d04 100644 --- a/iotdb-client/client-py/iotdb/dbapi/tests/test_cursor.py +++ b/iotdb-client/client-py/tests/integration/dbapi/test_cursor.py @@ -16,7 +16,7 @@ # under the License. # -from iotdb.IoTDBContainer import IoTDBContainer +from tests.integration.iotdb_container import IoTDBContainer from iotdb.dbapi import connect from iotdb.dbapi.Cursor import Cursor diff --git a/iotdb-client/client-py/iotdb/IoTDBContainer.py b/iotdb-client/client-py/tests/integration/iotdb_container.py similarity index 100% rename from iotdb-client/client-py/iotdb/IoTDBContainer.py rename to iotdb-client/client-py/tests/integration/iotdb_container.py diff --git a/iotdb-client/client-py/iotdb/sqlalchemy/tests/__init__.py b/iotdb-client/client-py/tests/integration/sqlalchemy/__init__.py similarity index 100% rename from iotdb-client/client-py/iotdb/sqlalchemy/tests/__init__.py rename to iotdb-client/client-py/tests/integration/sqlalchemy/__init__.py diff --git a/iotdb-client/client-py/tests/integration/sqlalchemy/test_dialect.py b/iotdb-client/client-py/tests/integration/sqlalchemy/test_dialect.py new file mode 100644 index 0000000000000..35a16ff366e63 --- /dev/null +++ b/iotdb-client/client-py/tests/integration/sqlalchemy/test_dialect.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import operator + +from sqlalchemy import create_engine, inspect +from sqlalchemy.dialects import registry +from sqlalchemy.orm import Session +from sqlalchemy.sql import text +from tests.integration.iotdb_container import IoTDBContainer + +final_flag = True +failed_count = 0 + + +def test_fail(): + global failed_count + global final_flag + final_flag = False + failed_count += 1 + + +def print_message(message): + print("*********") + print(message) + print("*********") + assert False + + +def test_dialect(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + url = ( + "iotdb://root:root@" + + db.get_container_host_ip() + + ":" + + db.get_exposed_port(6667) + ) + registry.register("iotdb", "iotdb.sqlalchemy.IoTDBDialect", "IoTDBDialect") + eng = create_engine(url) + + with Session(eng) as session: + session.execute(text("create database root.cursor")) + session.execute(text("create database root.cursor_s1")) + session.execute( + text( + "create timeseries root.cursor.device1.temperature with datatype=FLOAT,encoding=RLE" + ) + ) + session.execute( + text( + "create timeseries root.cursor.device1.status with datatype=FLOAT,encoding=RLE" + ) + ) + session.execute( + text( + "create timeseries root.cursor.device2.temperature with datatype=FLOAT,encoding=RLE" + ) + ) + + insp = inspect(eng) + # test get_schema_names + schema_names = insp.get_schema_names() + if not operator.ge(schema_names, ["root.cursor_s1", "root.cursor"]): + test_fail() + print_message("test get_schema_names failed!") + # test get_table_names + table_names = insp.get_table_names("root.cursor") + if not operator.eq(table_names, ["device1", "device2"]): + test_fail() + print_message("test get_table_names failed!") + # test get_columns + columns = insp.get_columns(table_name="device1", schema="root.cursor") + if len(columns) != 3: + test_fail() + print_message("test get_columns failed!") + + with Session(eng) as session: + session.execute(text("delete database root.cursor")) + session.execute(text("delete database root.cursor_s1")) + + # close engine + eng.dispose() + + +if final_flag: + print("All executions done!!") +else: + print("Some test failed, please have a check") + print("failed count: ", failed_count) + exit(1) diff --git a/iotdb-client/client-py/tests/integration/tablet_performance_comparison.py b/iotdb-client/client-py/tests/integration/tablet_performance_comparison.py index 3626e818a85d0..c22124ec00938 100644 --- a/iotdb-client/client-py/tests/integration/tablet_performance_comparison.py +++ b/iotdb-client/client-py/tests/integration/tablet_performance_comparison.py @@ -113,7 +113,9 @@ def create_open_session(): port_ = "6667" username_ = "root" password_ = "root" - session = Session(ip, port_, username_, password_, fetch_size=1024, zone_id="UTC+8") + session = Session( + ip, port_, username_, password_, fetch_size=1024, zone_id="Asia/Shanghai" + ) session.open(False) return session diff --git a/iotdb-client/client-py/tests/integration/test_aligned_timeseries.py b/iotdb-client/client-py/tests/integration/test_aligned_timeseries.py index f2a0389053883..2c36f9484276e 100644 --- a/iotdb-client/client-py/tests/integration/test_aligned_timeseries.py +++ b/iotdb-client/client-py/tests/integration/test_aligned_timeseries.py @@ -20,7 +20,7 @@ from iotdb.Session import Session from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor from iotdb.utils.Tablet import Tablet -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer # whether the test has passed final_flag = True diff --git a/iotdb-client/client-py/tests/integration/test_dataframe.py b/iotdb-client/client-py/tests/integration/test_dataframe.py index a93efdbea3e53..f314fbac18434 100644 --- a/iotdb-client/client-py/tests/integration/test_dataframe.py +++ b/iotdb-client/client-py/tests/integration/test_dataframe.py @@ -17,7 +17,7 @@ # from iotdb.Session import Session -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer from numpy.testing import assert_array_equal diff --git a/iotdb-client/client-py/tests/integration/test_delete_data.py b/iotdb-client/client-py/tests/integration/test_delete_data.py index 8f44574726495..cc4103947f7da 100644 --- a/iotdb-client/client-py/tests/integration/test_delete_data.py +++ b/iotdb-client/client-py/tests/integration/test_delete_data.py @@ -19,7 +19,7 @@ # Uncomment the following line to use apache-iotdb module installed by pip3 from iotdb.Session import Session -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer # whether the test has passed final_flag = True diff --git a/iotdb-client/client-py/tests/integration/test_new_data_types.py b/iotdb-client/client-py/tests/integration/test_new_data_types.py new file mode 100644 index 0000000000000..e59138fa0359d --- /dev/null +++ b/iotdb-client/client-py/tests/integration/test_new_data_types.py @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from datetime import date + +import numpy as np +import pandas as pd +from tzlocal import get_localzone_name + +from iotdb.Session import Session +from iotdb.SessionPool import PoolConfig, create_session_pool +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.Tablet import Tablet +from .iotdb_container import IoTDBContainer + + +def test_session(): + session_test() + + +def test_session_pool(): + session_test(True) + + +def session_test(use_session_pool=False): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + + if use_session_pool: + pool_config = PoolConfig( + db.get_container_host_ip(), + db.get_exposed_port(6667), + "root", + "root", + None, + 1024, + max_retry=3, + ) + session_pool = create_session_pool(pool_config, 1, 3000) + session = session_pool.get_session() + else: + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open(False) + + if not session.is_open(): + print("can't open session") + exit(1) + + device_id = "root.sg_test_01.d_04" + measurements_new_type = ["s_01", "s_02", "s_03", "s_04"] + data_types_new_type = [ + TSDataType.DATE, + TSDataType.TIMESTAMP, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_new_type = [ + [date(2024, 1, 1), 1, b"\x12\x34", "test01"], + [date(2024, 1, 2), 2, b"\x12\x34", "test02"], + [date(2024, 1, 3), 3, b"\x12\x34", "test03"], + [date(2024, 1, 4), 4, b"\x12\x34", "test04"], + ] + timestamps_new_type = [1, 2, 3, 4] + tablet_new_type = Tablet( + device_id, + measurements_new_type, + data_types_new_type, + values_new_type, + timestamps_new_type, + ) + session.insert_tablet(tablet_new_type) + np_values_new_type = [ + np.array( + [date(2024, 1, 5), date(2024, 1, 6), date(2024, 1, 7), date(2024, 1, 8)] + ), + np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()), + np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]), + np.array(["test05", "test06", "test07", "test08"]), + ] + np_timestamps_new_type = np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()) + np_tablet_new_type = NumpyTablet( + device_id, + measurements_new_type, + data_types_new_type, + np_values_new_type, + np_timestamps_new_type, + ) + session.insert_tablet(np_tablet_new_type) + session.insert_records( + [device_id, device_id], + [9, 10], + [measurements_new_type, measurements_new_type], + [data_types_new_type, data_types_new_type], + [ + [date(2024, 1, 9), 9, b"\x12\x34", "test09"], + [date(2024, 1, 10), 10, b"\x12\x34", "test010"], + ], + ) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + print(dataset.get_column_names()) + while dataset.has_next(): + print(dataset.next()) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + df = dataset.todf() + print(df.to_string()) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + cnt = 0 + while dataset.has_next(): + row_record = dataset.next() + timestamp = row_record.get_timestamp() + assert row_record.get_fields()[0].get_date_value() == date( + 2024, 1, timestamp + ) + assert row_record.get_fields()[1].get_object_value( + TSDataType.TIMESTAMP + ) == pd.Timestamp(timestamp, unit="ms", tz=get_localzone_name()) + assert row_record.get_fields()[2].get_binary_value() == b"\x12\x34" + assert row_record.get_fields()[3].get_string_value() == "test0" + str( + timestamp + ) + cnt += 1 + assert cnt == 10 + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + df = dataset.todf() + rows, columns = df.shape + assert rows == 10 + assert columns == 5 + + session.insert_records( + [device_id, device_id], + [11, 12], + [measurements_new_type, ["s_02", "s_03", "s_04"]], + [ + data_types_new_type, + [ + TSDataType.TIMESTAMP, + TSDataType.BLOB, + TSDataType.STRING, + ], + ], + [ + [date(1971, 1, 1), 11, b"\x12\x34", "test11"], + [12, b"\x12\x34", "test12"], + ], + ) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04 where time > 10" + ) as dataset: + cnt = 0 + while dataset.has_next(): + cnt += 1 + print(dataset.next()) + assert cnt == 2 + + # close session connection. + session.close() diff --git a/iotdb-client/client-py/tests/integration/test_one_device.py b/iotdb-client/client-py/tests/integration/test_one_device.py index ce4c02b0123e8..337445eb3b877 100644 --- a/iotdb-client/client-py/tests/integration/test_one_device.py +++ b/iotdb-client/client-py/tests/integration/test_one_device.py @@ -19,7 +19,7 @@ # Uncomment the following line to use apache-iotdb module installed by pip3 from iotdb.Session import Session -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer # whether the test has passed final_flag = True diff --git a/iotdb-client/client-py/tests/integration/test_relational_session.py b/iotdb-client/client-py/tests/integration/test_relational_session.py new file mode 100644 index 0000000000000..48c3cf307a47d --- /dev/null +++ b/iotdb-client/client-py/tests/integration/test_relational_session.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np + +from iotdb.table_session import TableSession, TableSessionConfig +from iotdb.table_session_pool import TableSessionPool, TableSessionPoolConfig +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.Tablet import Tablet, ColumnType +from .iotdb_container import IoTDBContainer + + +def test_session(): + session_test() + + +def test_session_pool(): + session_test(True) + + +def session_test(use_session_pool=False): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + if use_session_pool: + config = TableSessionPoolConfig( + node_urls=[f"{db.get_container_host_ip()}:{db.get_exposed_port(6667)}"] + ) + session_pool = TableSessionPool(config) + session = session_pool.get_session() + else: + config = TableSessionConfig( + node_urls=[f"{db.get_container_host_ip()}:{db.get_exposed_port(6667)}"] + ) + session = TableSession(config) + + session.execute_non_query_statement("CREATE DATABASE IF NOT EXISTS db1") + session.execute_non_query_statement('USE "db1"') + session.execute_non_query_statement( + "CREATE TABLE table5 (tag1 string tag, attr1 string attribute, " + + "m1 double " + + "field)" + ) + + column_names = [ + "tag1", + "attr1", + "m1", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.DOUBLE, + ] + column_types = [ColumnType.TAG, ColumnType.ATTRIBUTE, ColumnType.FIELD] + timestamps = [] + values = [] + for row in range(15): + timestamps.append(row) + values.append(["tag:" + str(row), "attr:" + str(row), row * 1.0]) + tablet = Tablet( + "table5", column_names, data_types, values, timestamps, column_types + ) + session.insert(tablet) + + session.execute_non_query_statement("FLush") + + np_timestamps = np.arange(15, 30, dtype=np.dtype(">i8")) + np_values = [ + np.array(["tag:{}".format(i) for i in range(15, 30)]), + np.array(["attr:{}".format(i) for i in range(15, 30)]), + np.linspace(15.0, 29.0, num=15, dtype=TSDataType.DOUBLE.np_dtype()), + ] + + np_tablet = NumpyTablet( + "table5", + column_names, + data_types, + np_values, + np_timestamps, + column_types=column_types, + ) + session.insert(np_tablet) + + with session.execute_query_statement( + "select * from table5 order by time" + ) as dataset: + cnt = 0 + while dataset.has_next(): + row_record = dataset.next() + timestamp = row_record.get_fields()[0].get_long_value() + assert ( + "tag:" + str(timestamp) + == row_record.get_fields()[1].get_string_value() + ) + assert ( + "attr:" + str(timestamp) + == row_record.get_fields()[2].get_string_value() + ) + assert timestamp * 1.0 == row_record.get_fields()[3].get_double_value() + cnt += 1 + assert 30 == cnt + + with session.execute_query_statement( + "select * from table5 order by time" + ) as dataset: + df = dataset.todf() + rows, columns = df.shape + assert rows == 30 + assert columns == 4 + + # close session connection. + session.close() diff --git a/iotdb-client/client-py/tests/integration/test_session.py b/iotdb-client/client-py/tests/integration/test_session.py index 197b9b9de522b..75cdf056113b4 100644 --- a/iotdb-client/client-py/tests/integration/test_session.py +++ b/iotdb-client/client-py/tests/integration/test_session.py @@ -25,7 +25,7 @@ from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor from iotdb.utils.NumpyTablet import NumpyTablet from iotdb.utils.Tablet import Tablet -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer # whether the test has passed final_flag = True @@ -85,11 +85,15 @@ def session_test(use_session_pool=False): session.set_storage_group("root.sg_test_03") session.set_storage_group("root.sg_test_04") - if session.delete_storage_group("root.sg_test_02") < 0: + try: + session.delete_storage_group("root.sg_test_02") + except Exception: test_fail() print_message("delete database failed") - if session.delete_storage_groups(["root.sg_test_03", "root.sg_test_04"]) < 0: + try: + session.delete_storage_groups(["root.sg_test_03", "root.sg_test_04"]) + except Exception: test_fail() print_message("delete databases failed") @@ -177,7 +181,7 @@ def session_test(use_session_pool=False): ) # delete time series - if ( + try: session.delete_time_series( [ "root.sg_test_01.d_01.s_07", @@ -185,8 +189,7 @@ def session_test(use_session_pool=False): "root.sg_test_01.d_01.s_09", ] ) - < 0 - ): + except Exception: test_fail() print_message("delete time series failed") @@ -220,12 +223,30 @@ def session_test(use_session_pool=False): TSDataType.DOUBLE, TSDataType.TEXT, ] - if ( + try: session.insert_record( "root.sg_test_01.d_01", 1, measurements_, data_types_, values_ ) - < 0 - ): + except Exception: + test_fail() + print_message("insert record failed") + + # insert one record with none into the database. + measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] + values_ = [False, 10, 11, None, None, None] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + ] + try: + session.insert_record( + "root.sg_test_01.d_01", 1, measurements_, data_types_, values_ + ) + except Exception: test_fail() print_message("insert record failed") @@ -240,12 +261,44 @@ def session_test(use_session_pool=False): ] data_type_list_ = [data_types_, data_types_] device_ids_ = ["root.sg_test_01.d_01", "root.sg_test_01.d_02"] - if ( + try: session.insert_records( device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ ) - < 0 - ): + except Exception: + test_fail() + print_message("insert records failed") + + # insert multiple records with none into database + measurements_list_ = [ + ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], + ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"], + ] + values_list_ = [ + [False, 22, 33, 4.4, 55.1, "test_records01"], + [None, None, None, None, 8.125, bytes("test_records02", "utf-8")], + ] + data_type_list_ = [data_types_, data_types_] + device_ids_ = ["root.sg_test_01.d_01", "root.sg_test_01.d_02"] + try: + session.insert_records( + device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ + ) + except Exception: + test_fail() + print_message("insert records failed") + + values_list_ = [ + [False, 22, 33, 4.4, 55.1, "test_records01"], + [None, None, None, None, None, None], + ] + data_type_list_ = [data_types_, data_types_] + device_ids_ = ["root.sg_test_01.d_01", "root.sg_test_01.d_02"] + try: + session.insert_records( + device_ids_, [2, 3], measurements_list_, data_type_list_, values_list_ + ) + except Exception: test_fail() print_message("insert records failed") @@ -261,7 +314,9 @@ def session_test(use_session_pool=False): "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ ) - if session.insert_tablet(tablet_) < 0: + try: + session.insert_tablet(tablet_) + except Exception: test_fail() print_message("insert tablet failed") @@ -282,7 +337,9 @@ def session_test(use_session_pool=False): np_values_, np_timestamps_, ) - if session.insert_tablet(np_tablet_) < 0: + try: + session.insert_tablet(np_tablet_) + except Exception: test_fail() print_message("insert numpy tablet failed") @@ -297,7 +354,9 @@ def session_test(use_session_pool=False): values_, [12, 13, 14, 15], ) - if session.insert_tablets([tablet_01, tablet_02]) < 0: + try: + session.insert_tablets([tablet_01, tablet_02]) + except Exception: test_fail() print_message("insert tablets failed") @@ -312,7 +371,9 @@ def session_test(use_session_pool=False): tablet_ = Tablet( "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ ) - if session.insert_tablet(tablet_) < 0: + try: + session.insert_tablet(tablet_) + except Exception: test_fail() print_message("insert tablet with empty cells failed") @@ -342,7 +403,9 @@ def session_test(use_session_pool=False): np_timestamps_, np_bitmaps_, ) - if session.insert_tablet(np_tablet_) < 0: + try: + session.insert_tablet(np_tablet_) + except Exception: test_fail() print_message("insert numpy tablet with empty cells failed") @@ -360,7 +423,7 @@ def session_test(use_session_pool=False): ] values_list = [[False, 22, 33], [True, 1, 23], [False, 15, 26]] - if ( + try: session.insert_records_of_one_device( "root.sg_test_01.d_01", time_list, @@ -368,8 +431,7 @@ def session_test(use_session_pool=False): data_types_list, values_list, ) - < 0 - ): + except Exception: test_fail() print_message("insert records of one device failed") diff --git a/iotdb-client/client-py/tests/integration/test_session_pool.py b/iotdb-client/client-py/tests/integration/test_session_pool.py index 43351be251690..f85ffa80eb475 100644 --- a/iotdb-client/client-py/tests/integration/test_session_pool.py +++ b/iotdb-client/client-py/tests/integration/test_session_pool.py @@ -17,7 +17,7 @@ # from threading import Thread -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer from iotdb.SessionPool import create_session_pool, PoolConfig CONTAINER_NAME = "iotdb:dev" diff --git a/iotdb-client/client-py/tests/integration/test_tablemodel_insert.py b/iotdb-client/client-py/tests/integration/test_tablemodel_insert.py new file mode 100644 index 0000000000000..70f9730e2d003 --- /dev/null +++ b/iotdb-client/client-py/tests/integration/test_tablemodel_insert.py @@ -0,0 +1,1592 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np +from iotdb.table_session import TableSession, TableSessionConfig +from iotdb.utils.BitMap import BitMap +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.Tablet import Tablet, ColumnType +from iotdb.utils.NumpyTablet import NumpyTablet +from datetime import date +from .iotdb_container import IoTDBContainer + + +# Test inserting tablet data +def test_insert_use_tablet(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + config = TableSessionConfig( + node_urls=[f"{db.get_container_host_ip()}:{db.get_exposed_port(6667)}"] + ) + session = TableSession(config) + + # Preparation before testing + session.execute_non_query_statement( + "create database test_insert_relational_tablet_tablet" + ) + session.execute_non_query_statement("use test_insert_relational_tablet_tablet") + session.execute_non_query_statement( + "create table table_a(" + '"地区" STRING TAG, "厂号" STRING TAG, "设备号" STRING TAG, ' + '"日期" string attribute, "时间" string attribute, "负责人" string attribute,' + '"测点1" BOOLEAN FIELD, "测点2" INT32 FIELD, "测点3" INT64 FIELD, "测点4" FLOAT FIELD, "测点5" DOUBLE FIELD,' + '"测点6" TEXT FIELD, "测点7" TIMESTAMP FIELD, "测点8" DATE FIELD, "测点9" BLOB FIELD, "测点10" STRING FIELD)' + ) + session.execute_non_query_statement( + "create table table_b(" + "tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG, " + "attr1 string attribute, attr2 string attribute, attr3 string attribute," + "BOOLEAN BOOLEAN FIELD, INT32 INT32 FIELD, INT64 INT64 FIELD, FLOAT FLOAT FIELD, DOUBLE DOUBLE FIELD," + "TEXT TEXT FIELD, TIMESTAMP TIMESTAMP FIELD, DATE DATE FIELD, BLOB BLOB FIELD, STRING STRING FIELD)" + ) + + # 1、General scenario + expect = 10 + table_name = "table_b" + column_names = [ + "tag1", + "tag2", + "tag3", + "attr1", + "attr2", + "attr3", + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TIMESTAMP", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + timestamps = [] + values = [] + for row_b in range(10): + timestamps.append(row_b) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + tablet = Tablet( + table_name, column_names, data_types, values, timestamps, column_types + ) + session.insert(tablet) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from table_b" + ) as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 2、Including Chinese scenes + expect = 10 + table_name = "table_a" + column_names = [ + "地区", + "厂号", + "设备号", + "日期", + "时间", + "负责人", + "测点1", + "测点2", + "测点3", + "测点4", + "测点5", + "测点6", + "测点7", + "测点8", + "测点9", + "测点10", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + timestamps = [] + values = [] + for row_a in range(10): + timestamps.append(row_a) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ] + ) + values.append( + [ + "tag1：" + str(row_a), + "tag2：" + str(row_a), + "tag3：" + str(row_a), + "attr1:" + str(row_a), + "attr2:" + str(row_a), + "attr3:" + str(row_a), + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + tablet = Tablet( + table_name, column_names, data_types, values, timestamps, column_types + ) + session.insert(tablet) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from table_a" + ) as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + session.close() + + +# Test inserting NumpyTablet data +def test_insert_relational_tablet_use_numpy_tablet(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + config = TableSessionConfig( + node_urls=[f"{db.get_container_host_ip()}:{db.get_exposed_port(6667)}"] + ) + session = TableSession(config) + + # Preparation before testing + session.execute_non_query_statement( + "create database test_insert_relational_tablet_use_numpy_tablet" + ) + session.execute_non_query_statement( + "use test_insert_relational_tablet_use_numpy_tablet" + ) + session.execute_non_query_statement( + "create table table_b(" + "tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG, " + "attr1 string attribute, attr2 string attribute, attr3 string attribute," + "BOOLEAN BOOLEAN FIELD, INT32 INT32 FIELD, INT64 INT64 FIELD, FLOAT FLOAT FIELD, DOUBLE DOUBLE FIELD," + "TEXT TEXT FIELD, TIMESTAMP TIMESTAMP FIELD, DATE DATE FIELD, BLOB BLOB FIELD, STRING STRING FIELD)" + ) + session.execute_non_query_statement( + "create table table_d(" + "tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG, " + "attr1 string attribute, attr2 string attribute, attr3 string attribute," + "BOOLEAN BOOLEAN FIELD, INT32 INT32 FIELD, INT64 INT64 FIELD, FLOAT FLOAT FIELD, DOUBLE DOUBLE FIELD," + "TEXT TEXT FIELD, TIMESTAMP TIMESTAMP FIELD, DATE DATE FIELD, BLOB BLOB FIELD, STRING STRING FIELD)" + ) + + # 1、No null + expect = 10 + table_name = "table_b" + column_names = [ + "tag1", + "tag2", + "tag3", + "attr1", + "attr2", + "attr3", + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TIMESTAMP", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + np_timestamps = np.arange(0, 10, dtype=np.dtype(">i8")) + np_values = [ + np.array(["tag1:{}".format(i) for i in range(0, 10)]), + np.array(["tag2:{}".format(i) for i in range(0, 10)]), + np.array(["tag3:{}".format(i) for i in range(0, 10)]), + np.array(["attr1:{}".format(i) for i in range(0, 10)]), + np.array(["attr2:{}".format(i) for i in range(0, 10)]), + np.array(["attr3:{}".format(i) for i in range(0, 10)]), + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_tablet = NumpyTablet( + table_name, + column_names, + data_types, + np_values, + np_timestamps, + column_types=column_types, + ) + session.insert(np_tablet) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from table_b" + ) as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 2、Contains null + expect = 10 + table_name = "table_d" + column_names = [ + "tag1", + "tag2", + "tag3", + "attr1", + "attr2", + "attr3", + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TIMESTAMP", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + np_timestamps = np.arange(0, 10, dtype=np.dtype(">i8")) + np_values = [ + np.array(["tag1:{}".format(i) for i in range(0, 10)]), + np.array(["tag2:{}".format(i) for i in range(0, 10)]), + np.array(["tag3:{}".format(i) for i in range(0, 10)]), + np.array(["attr1:{}".format(i) for i in range(0, 10)]), + np.array(["attr2:{}".format(i) for i in range(0, 10)]), + np.array(["attr3:{}".format(i) for i in range(0, 10)]), + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_bitmaps_ = [] + for i in range(len(column_types)): + np_bitmaps_.append(BitMap(len(np_timestamps))) + np_bitmaps_[5].mark(9) + np_bitmaps_[6].mark(8) + np_bitmaps_[7].mark(9) + np_bitmaps_[8].mark(8) + np_bitmaps_[9].mark(9) + np_bitmaps_[10].mark(8) + np_bitmaps_[11].mark(9) + np_bitmaps_[12].mark(8) + np_bitmaps_[13].mark(9) + np_bitmaps_[14].mark(8) + np_tablet = NumpyTablet( + table_name, + column_names, + data_types, + np_values, + np_timestamps, + bitmaps=np_bitmaps_, + column_types=column_types, + ) + session.insert(np_tablet) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from table_d" + ) as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + session.close() + + +# Test automatic creation +def test_insert_relational_tablet_auto_create(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + config = TableSessionConfig( + node_urls=[f"{db.get_container_host_ip()}:{db.get_exposed_port(6667)}"] + ) + session = TableSession(config) + + # Preparation before testing + session.execute_non_query_statement( + "create database test_insert_relational_tablet_use_numpy_tablet" + ) + session.execute_non_query_statement( + "use test_insert_relational_tablet_use_numpy_tablet" + ) + + # 1、Test inserting tablet data(Insert 10 times) + for i in range(1, 10): + table_name = "t" + str(i) + column_names = [ + "tag1", + "tag2", + "tag3", + "attr1", + "attr2", + "attr3", + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TIMESTAMP", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + timestamps = [] + values = [] + for row in range(10): + timestamps.append(row) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ] + ) + values.append( + [ + "tag1：" + str(row), + "tag2：" + str(row), + "tag3：" + str(row), + "attr1:" + str(row), + "attr2:" + str(row), + "attr3:" + str(row), + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + tablet = Tablet( + table_name, column_names, data_types, values, timestamps, column_types + ) + session.insert(tablet) + + # 2、Test inserting NumpyTablet data(Insert 10 times) + for i in range(1, 10): + table_name = "t" + str(i) + column_names = [ + "tag1", + "tag2", + "tag3", + "attr1", + "attr2", + "attr3", + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TIMESTAMP", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + np_timestamps = np.arange(0, 10, dtype=np.dtype(">i8")) + np_values = [ + np.array(["tag1:{}".format(i) for i in range(0, 10)]), + np.array(["tag2:{}".format(i) for i in range(0, 10)]), + np.array(["tag3:{}".format(i) for i in range(0, 10)]), + np.array(["attr1:{}".format(i) for i in range(0, 10)]), + np.array(["attr2:{}".format(i) for i in range(0, 10)]), + np.array(["attr3:{}".format(i) for i in range(0, 10)]), + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_tablet = NumpyTablet( + table_name, + column_names, + data_types, + np_values, + np_timestamps, + column_types=column_types, + ) + session.insert(np_tablet) + + session.close() diff --git a/iotdb-client/client-py/tests/integration/test_tablemodel_query.py b/iotdb-client/client-py/tests/integration/test_tablemodel_query.py new file mode 100644 index 0000000000000..d1ec27fbaf4d4 --- /dev/null +++ b/iotdb-client/client-py/tests/integration/test_tablemodel_query.py @@ -0,0 +1,476 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import math + +import pandas as pd +from tzlocal import get_localzone_name + +from iotdb.table_session import TableSession, TableSessionConfig +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.Tablet import Tablet, ColumnType +from datetime import date + +from iotdb.utils.rpc_utils import convert_to_timestamp +from .iotdb_container import IoTDBContainer + + +# Test query data +def test_query_data(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + config = TableSessionConfig( + node_urls=[f"{db.get_container_host_ip()}:{db.get_exposed_port(6667)}"] + ) + session = TableSession(config) + + # Preparation before testing + session.execute_non_query_statement( + "create database test_insert_relational_tablet_tablet" + ) + session.execute_non_query_statement("use test_insert_relational_tablet_tablet") + session.execute_non_query_statement( + "create table table_b(" + "tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG, " + "attr1 string attribute, attr2 string attribute, attr3 string attribute," + "BOOLEAN BOOLEAN FIELD, INT32 INT32 FIELD, INT64 INT64 FIELD, FLOAT FLOAT FIELD, DOUBLE DOUBLE FIELD," + "TEXT TEXT FIELD, TIMESTAMP TIMESTAMP FIELD, DATE DATE FIELD, BLOB BLOB FIELD, STRING STRING FIELD)" + ) + + # 1、General scenario + expect = 10 + table_name = "table_b" + column_names = [ + "tag1", + "tag2", + "tag3", + "attr1", + "attr2", + "attr3", + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TIMESTAMP", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + column_types = [ + ColumnType.TAG, + ColumnType.TAG, + ColumnType.TAG, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.ATTRIBUTE, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ColumnType.FIELD, + ] + timestamps = [] + values = [] + for row_b in range(10): + timestamps.append(row_b) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ] + ) + values.append( + [ + "tag1：" + str(row_b), + "tag2：" + str(row_b), + "tag3：" + str(row_b), + "attr1:" + str(row_b), + "attr2:" + str(row_b), + "attr3:" + str(row_b), + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ] + ) + tablet = Tablet( + table_name, column_names, data_types, values, timestamps, column_types + ) + session.insert(tablet) + row = 0 + with session.execute_query_statement( + "select * from table_b" + ) as session_data_set: + print(session_data_set.get_column_names()) + while session_data_set.has_next(): + print(session_data_set.next()) + row += 1 + # Determine whether it meets expectations + assert expect == row + + with session.execute_query_statement( + "select * from table_b" + ) as session_data_set: + df = session_data_set.todf() + rows, columns = df.shape + assert rows == expect + assert columns == len(column_names) + 1 + + row = 0 + with session.execute_query_statement( + "select " + ",".join(column_names) + " from table_b" + ) as session_data_set: + assert session_data_set.get_column_names() == column_names + while session_data_set.has_next(): + row_record = session_data_set.next() + assert row_record.get_timestamp() == 0 + for i in range(len(column_names)): + if values[row][i] is not None and ( + data_types[i] == TSDataType.FLOAT + or data_types[i] == TSDataType.DOUBLE + ): + assert math.isclose( + row_record.get_fields()[i].get_object_value(data_types[i]), + values[row][i], + rel_tol=1e-6, + ) + elif data_types[i] == TSDataType.TIMESTAMP: + actual = row_record.get_fields()[i].get_timestamp_value() + expected = convert_to_timestamp( + values[row][i], "ms", get_localzone_name() + ) + if pd.isnull(actual): + assert pd.isnull(expected) + else: + assert actual == expected + else: + assert ( + row_record.get_fields()[i].get_object_value(data_types[i]) + == values[row][i] + ) + row += 1 + # Determine whether it meets expectations + assert expect == row + + with session.execute_query_statement( + "select " + ",".join(column_names) + " from table_b" + ) as session_data_set: + df = session_data_set.todf() + assert list(df.columns) == column_names + rows, columns = df.shape + assert rows == expect + assert columns == len(column_names) + for i in range(rows): + for j in range(columns): + if pd.isna(df.iloc[i, j]): + continue + elif isinstance(values[i][j], float): + assert math.isclose( + df.iloc[i, j], + values[i][j], + rel_tol=1e-6, + ) + elif isinstance(df.iloc[i, j], pd.Timestamp): + actual = df.iloc[i, j] + expected = pd.Series( + convert_to_timestamp( + values[i][j], "ms", get_localzone_name() + ) + )[0] + assert actual == expected + else: + assert df.iloc[i, j] == values[i][j] + + row = 0 + with session.execute_query_statement( + "select tag1, tag1 from table_b" + ) as session_data_set: + assert session_data_set.get_column_names() == ["tag1", "tag1"] + while session_data_set.has_next(): + row_record = session_data_set.next() + assert row_record.get_timestamp() == 0 + for i in range(len(["tag1", "tag1"])): + assert ( + row_record.get_fields()[i].get_object_value( + row_record.get_fields()[i].get_data_type() + ) + == values[row][0] + ) + row += 1 + # Determine whether it meets expectations + assert expect == row + + with session.execute_query_statement( + "select tag1, tag1 from table_b" + ) as session_data_set: + df = session_data_set.todf() + assert list(df.columns) == ["tag1", "tag1"] + rows, columns = df.shape + assert rows == expect + assert columns == 2 + for i in range(rows): + for j in range(columns): + if pd.isna(df.iloc[i, j]): + assert values[i][j] is None + else: + assert df.iloc[i, j] == values[i][0] + + row = 0 + with session.execute_query_statement( + "select tag1, attr1 as tag1 from table_b" + ) as session_data_set: + assert session_data_set.get_column_names() == ["tag1", "tag1"] + while session_data_set.has_next(): + row_record = session_data_set.next() + print(row_record) + assert row_record.get_timestamp() == 0 + assert ( + row_record.get_fields()[0].get_object_value( + row_record.get_fields()[0].get_data_type() + ) + == values[row][0] + ) + assert ( + row_record.get_fields()[1].get_object_value( + row_record.get_fields()[1].get_data_type() + ) + == values[row][3] + ) + row += 1 + # Determine whether it meets expectations + assert expect == row + + with session.execute_query_statement( + "select tag1, attr1 as tag1 from table_b" + ) as session_data_set: + df = session_data_set.todf() + assert list(df.columns) == ["tag1", "tag1"] + rows, columns = df.shape + assert rows == expect + assert columns == 2 + for i in range(rows): + assert df.iloc[i, 0] == values[i][0] + assert df.iloc[i, 1] == values[i][3] + + session.close() diff --git a/iotdb-client/client-py/tests/integration/test_tablet.py b/iotdb-client/client-py/tests/integration/test_tablet.py index 8a035318d9f3e..f91c1bc6dd645 100644 --- a/iotdb-client/client-py/tests/integration/test_tablet.py +++ b/iotdb-client/client-py/tests/integration/test_tablet.py @@ -19,7 +19,7 @@ import pandas as pd from pandas.testing import assert_frame_equal -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer from iotdb.Session import Session from iotdb.utils.IoTDBConstants import TSDataType from iotdb.utils.Tablet import Tablet diff --git a/iotdb-client/client-py/tests/integration/test_todf.py b/iotdb-client/client-py/tests/integration/test_todf.py index 03e97974776a7..9cd8ad26e6bad 100644 --- a/iotdb-client/client-py/tests/integration/test_todf.py +++ b/iotdb-client/client-py/tests/integration/test_todf.py @@ -22,7 +22,7 @@ import pandas as pd from pandas.testing import assert_frame_equal -from iotdb.IoTDBContainer import IoTDBContainer +from .iotdb_container import IoTDBContainer from iotdb.Session import Session from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor from iotdb.utils.Tablet import Tablet diff --git a/iotdb-client/client-py/tests/integration/test_treemodel_insert.py b/iotdb-client/client-py/tests/integration/test_treemodel_insert.py new file mode 100644 index 0000000000000..9feda17133f9e --- /dev/null +++ b/iotdb-client/client-py/tests/integration/test_treemodel_insert.py @@ -0,0 +1,6027 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np +from .iotdb_container import IoTDBContainer +from iotdb.Session import Session +from iotdb.utils.Tablet import Tablet +from datetime import date +from iotdb.utils.BitMap import BitMap +from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor +from iotdb.utils.NumpyTablet import NumpyTablet + + +# Test writing a tablet data to a non aligned time series +def test_insert_tablet(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test1.g1") + ts_path_lst = [ + "root.test1.g1.d1.BOOLEAN", + "root.test1.g1.d1.INT32", + "root.test1.g1.d1.INT64", + "root.test1.g1.d1.FLOAT", + "root.test1.g1.d1.DOUBLE", + "root.test1.g1.d1.TEXT", + "root.test1.g1.d1.TS", + "root.test1.g1.d1.DATE", + "root.test1.g1.d1.BLOB", + "root.test1.g1.d1.STRING", + "root.test1.g1.d2.BOOLEAN", + "root.test1.g1.d2.INT32", + "root.test1.g1.d2.INT64", + "root.test1.g1.d2.FLOAT", + "root.test1.g1.d2.DOUBLE", + "root.test1.g1.d2.TEXT", + "root.test1.g1.d2.TS", + "root.test1.g1.d2.DATE", + "root.test1.g1.d2.BLOB", + "root.test1.g1.d2.STRING", + "root.test1.g1.d3.BOOLEAN", + "root.test1.g1.d3.INT32", + "root.test1.g1.d3.INT64", + "root.test1.g1.d3.FLOAT", + "root.test1.g1.d3.DOUBLE", + "root.test1.g1.d3.TEXT", + "root.test1.g1.d3.TS", + "root.test1.g1.d3.DATE", + "root.test1.g1.d3.BLOB", + "root.test1.g1.d3.STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_multi_time_series( + ts_path_lst, + data_type_lst, + encoding_lst, + compressor_lst, + props_lst=None, + tags_lst=None, + attributes_lst=None, + alias_lst=None, + ) + + # 1、Inserting tablet data + expect = 10 + device_id = "root.test1.g1.d1" + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_ = [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test101", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test101", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ] + timestamps_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + tablet_ = Tablet(device_id, measurements_, data_types_, values_, timestamps_) + session.insert_tablet(tablet_) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test1.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 2、Inserting Numpy Tablet data (No null) + expect = 10 + device_id = "root.test1.g1.d2" + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test101", + "test101", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test101", + "test101", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + np_tablet_ = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + session.insert_tablet(np_tablet_) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test1.g1.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 3、Inserting Numpy Tablet data (Contains null) + expect = 10 + device_id = "root.test1.g1.d3" + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, False], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + np_bitmaps_ = [] + for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) + np_bitmaps_[0].mark(9) + np_bitmaps_[1].mark(8) + np_bitmaps_[2].mark(9) + np_bitmaps_[4].mark(8) + np_bitmaps_[5].mark(9) + np_bitmaps_[6].mark(8) + np_bitmaps_[7].mark(9) + np_bitmaps_[8].mark(8) + np_bitmaps_[9].mark(9) + np_tablet_ = NumpyTablet( + device_id, + measurements_, + data_types_, + np_values_, + np_timestamps_, + np_bitmaps_, + ) + session.insert_tablet(np_tablet_) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test1.g1.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing a tablet data to align the time series +def test_insert_aligned_tablet(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test2.g3") + device_id = "root.test2.g3.d1" + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_aligned_time_series( + device_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device_id = "root.test2.g3.d2" + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_aligned_time_series( + device_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device_id = "root.test2.g3.d3" + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_aligned_time_series( + device_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + + # 1、Inserting tablet data + expect = 10 + device_id = "root.test2.g3.d1" + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_ = [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ] + timestamps_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + tablet_ = Tablet(device_id, measurements_, data_types_, values_, timestamps_) + session.insert_aligned_tablet(tablet_) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test2.g3.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 2、Inserting Numpy Tablet data (No null) + expect = 10 + device_id = "root.test2.g3.d2" + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + np_tablet_ = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + session.insert_aligned_tablet(np_tablet_) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test2.g3.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 3、Inserting Numpy Tablet data (Contains null) + expect = 10 + device_id = "root.test2.g3.d3" + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, False], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + np_bitmaps_ = [] + for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) + np_bitmaps_[0].mark(9) + np_bitmaps_[1].mark(8) + np_bitmaps_[2].mark(9) + np_bitmaps_[4].mark(8) + np_bitmaps_[5].mark(9) + np_bitmaps_[6].mark(8) + np_bitmaps_[7].mark(9) + np_bitmaps_[8].mark(8) + np_bitmaps_[9].mark(9) + np_tablet_ = NumpyTablet( + device_id, + measurements_, + data_types_, + np_values_, + np_timestamps_, + np_bitmaps_, + ) + session.insert_aligned_tablet(np_tablet_) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test2.g3.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing multiple tablet data to non aligned time series +def test_insert_tablets(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test3.g1") + session.set_storage_group("root.test3.g2") + ts_path_lst = [ + "root.test3.g1.d1.BOOLEAN", + "root.test3.g1.d1.INT32", + "root.test3.g1.d1.INT64", + "root.test3.g1.d1.FLOAT", + "root.test3.g1.d1.DOUBLE", + "root.test3.g1.d1.TEXT", + "root.test3.g1.d1.TS", + "root.test3.g1.d1.DATE", + "root.test3.g1.d1.BLOB", + "root.test3.g1.d1.STRING", + "root.test3.g1.d2.BOOLEAN", + "root.test3.g1.d2.INT32", + "root.test3.g1.d2.INT64", + "root.test3.g1.d2.FLOAT", + "root.test3.g1.d2.DOUBLE", + "root.test3.g1.d2.TEXT", + "root.test3.g1.d2.TS", + "root.test3.g1.d2.DATE", + "root.test3.g1.d2.BLOB", + "root.test3.g1.d2.STRING", + "root.test3.g1.d3.BOOLEAN", + "root.test3.g1.d3.INT32", + "root.test3.g1.d3.INT64", + "root.test3.g1.d3.FLOAT", + "root.test3.g1.d3.DOUBLE", + "root.test3.g1.d3.TEXT", + "root.test3.g1.d3.TS", + "root.test3.g1.d3.DATE", + "root.test3.g1.d3.BLOB", + "root.test3.g1.d3.STRING", + "root.test3.g2.d1.BOOLEAN", + "root.test3.g2.d1.INT32", + "root.test3.g2.d1.INT64", + "root.test3.g2.d1.FLOAT", + "root.test3.g2.d1.DOUBLE", + "root.test3.g2.d1.TEXT", + "root.test3.g2.d1.TS", + "root.test3.g2.d1.DATE", + "root.test3.g2.d1.BLOB", + "root.test3.g2.d1.STRING", + "root.test3.g2.d2.BOOLEAN", + "root.test3.g2.d2.INT32", + "root.test3.g2.d2.INT64", + "root.test3.g2.d2.FLOAT", + "root.test3.g2.d2.DOUBLE", + "root.test3.g2.d2.TEXT", + "root.test3.g2.d2.TS", + "root.test3.g2.d2.DATE", + "root.test3.g2.d2.BLOB", + "root.test3.g2.d2.STRING", + "root.test3.g2.d3.BOOLEAN", + "root.test3.g2.d3.INT32", + "root.test3.g2.d3.INT64", + "root.test3.g2.d3.FLOAT", + "root.test3.g2.d3.DOUBLE", + "root.test3.g2.d3.TEXT", + "root.test3.g2.d3.TS", + "root.test3.g2.d3.DATE", + "root.test3.g2.d3.BLOB", + "root.test3.g2.d3.STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_multi_time_series( + ts_path_lst, + data_type_lst, + encoding_lst, + compressor_lst, + props_lst=None, + tags_lst=None, + attributes_lst=None, + alias_lst=None, + ) + + # 1、Inserting tablet data + expect = 30 + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_ = [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ] + timestamps_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + device1_id = "root.test3.g1.d1" + device2_id = "root.test3.g1.d2" + device3_id = "root.test3.g1.d3" + tablet1_ = Tablet(device1_id, measurements_, data_types, values_, timestamps_) + tablet2_ = Tablet(device2_id, measurements_, data_types, values_, timestamps_) + tablet3_ = Tablet(device3_id, measurements_, data_types, values_, timestamps_) + tablet_lst = [tablet1_, tablet2_, tablet3_] + session.insert_tablets(tablet_lst) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test3.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test3.g1.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test3.g1.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 2、Inserting Numpy Tablet data + expect = 30 + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + device1_id = "root.test3.g2.d1" + device2_id = "root.test3.g2.d2" + device3_id = "root.test3.g2.d3" + np_tablet1_ = NumpyTablet( + device1_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + np_tablet2_ = NumpyTablet( + device2_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + np_tablet3_ = NumpyTablet( + device3_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + np_tablet_list = [np_tablet1_, np_tablet2_, np_tablet3_] + + session.insert_tablets(np_tablet_list) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test3.g2.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test3.g2.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test3.g2.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing multiple tablet data to aligned time series +def test_insert_aligned_tablets(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test4.g1") + session.set_storage_group("root.test4.g2") + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + device11_id = "root.test4.g1.d1" + session.create_aligned_time_series( + device11_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device12_id = "root.test4.g1.d2" + session.create_aligned_time_series( + device12_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device13_id = "root.test4.g1.d3" + session.create_aligned_time_series( + device13_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device21_id = "root.test4.g2.d1" + session.create_aligned_time_series( + device21_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device22_id = "root.test4.g2.d2" + session.create_aligned_time_series( + device22_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device23_id = "root.test4.g2.d3" + session.create_aligned_time_series( + device23_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + + # 1、Inserting tablet data + expect = 30 + + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_ = [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ] + timestamps_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + device1_id = "root.test4.g1.d1" + device2_id = "root.test4.g1.d2" + device3_id = "root.test4.g1.d3" + tablet1_ = Tablet(device1_id, measurements_, data_types, values_, timestamps_) + tablet2_ = Tablet(device2_id, measurements_, data_types, values_, timestamps_) + tablet3_ = Tablet(device3_id, measurements_, data_types, values_, timestamps_) + tablet_lst = [tablet1_, tablet2_, tablet3_] + session.insert_aligned_tablets(tablet_lst) + + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test4.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test4.g1.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test4.g1.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + + # 2、Inserting Numpy Tablet data + expect = 30 + + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + device1_id = "root.test4.g2.d1" + device2_id = "root.test4.g2.d2" + device_id = "root.test4.g2.d3" + np_tablet1_ = NumpyTablet( + device1_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + np_tablet2_ = NumpyTablet( + device2_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + np_tablet3_ = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + np_tablet_list = [np_tablet1_, np_tablet2_, np_tablet3_] + session.insert_aligned_tablets(np_tablet_list) + + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test4.g2.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test4.g2.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test4.g2.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing a Record data to a non aligned time series +def test_insert_record(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test5.g1") + ts_path_lst = [ + "root.test5.g1.d1.BOOLEAN", + "root.test5.g1.d1.INT32", + "root.test5.g1.d1.INT64", + "root.test5.g1.d1.FLOAT", + "root.test5.g1.d1.DOUBLE", + "root.test5.g1.d1.TEXT", + "root.test5.g1.d1.TS", + "root.test5.g1.d1.DATE", + "root.test5.g1.d1.BLOB", + "root.test5.g1.d1.STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_multi_time_series( + ts_path_lst, + data_type_lst, + encoding_lst, + compressor_lst, + props_lst=None, + tags_lst=None, + attributes_lst=None, + alias_lst=None, + ) + + expect = 9 + session.insert_record( + "root.test5.g1.d1", + 1, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 2, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 3, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 4, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 5, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 6, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 7, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 9, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ) + session.insert_record( + "root.test5.g1.d1", + 10, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test5.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing a Record data to align the time series +def test_insert_aligned_record(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test6.g1") + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + device11_id = "root.test6.g1.d1" + session.create_aligned_time_series( + device11_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + + expect = 9 + session.insert_aligned_record( + "root.test6.g1.d1", + 1, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 2, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 3, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 4, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 5, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 6, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 7, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 9, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ) + session.insert_aligned_record( + "root.test6.g1.d1", + 10, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test6.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing multiple Record data to non aligned time series +def test_insert_records(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test7.g1") + ts_path_lst = [ + "root.test7.g1.d1.BOOLEAN", + "root.test7.g1.d1.INT32", + "root.test7.g1.d1.INT64", + "root.test7.g1.d1.FLOAT", + "root.test7.g1.d1.DOUBLE", + "root.test7.g1.d1.TEXT", + "root.test7.g1.d1.TS", + "root.test7.g1.d1.DATE", + "root.test7.g1.d1.BLOB", + "root.test7.g1.d1.STRING", + "root.test7.g1.d2.BOOLEAN", + "root.test7.g1.d2.INT32", + "root.test7.g1.d2.INT64", + "root.test7.g1.d2.FLOAT", + "root.test7.g1.d2.DOUBLE", + "root.test7.g1.d2.TEXT", + "root.test7.g1.d2.TS", + "root.test7.g1.d2.DATE", + "root.test7.g1.d2.BLOB", + "root.test7.g1.d2.STRING", + "root.test7.g1.d3.BOOLEAN", + "root.test7.g1.d3.INT32", + "root.test7.g1.d3.INT64", + "root.test7.g1.d3.FLOAT", + "root.test7.g1.d3.DOUBLE", + "root.test7.g1.d3.TEXT", + "root.test7.g1.d3.TS", + "root.test7.g1.d3.DATE", + "root.test7.g1.d3.BLOB", + "root.test7.g1.d3.STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_multi_time_series( + ts_path_lst, + data_type_lst, + encoding_lst, + compressor_lst, + props_lst=None, + tags_lst=None, + attributes_lst=None, + alias_lst=None, + ) + + expect = 9 + session.insert_records( + [ + "root.test7.g1.d1", + "root.test7.g1.d2", + "root.test7.g1.d3", + "root.test7.g1.d1", + "root.test7.g1.d2", + "root.test7.g1.d3", + "root.test7.g1.d1", + "root.test7.g1.d2", + "root.test7.g1.d3", + ], + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [ + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + ], + [ + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + ], + [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ], + ) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test7.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test7.g1.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test7.g1.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writing multiple Record data to aligned time series +def test_insert_aligned_records(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test8.g1") + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + device11_id = "root.test8.g1.d1" + session.create_aligned_time_series( + device11_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device11_id = "root.test8.g1.d2" + session.create_aligned_time_series( + device11_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + device11_id = "root.test8.g1.d3" + session.create_aligned_time_series( + device11_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + + expect = 9 + session.insert_aligned_records( + [ + "root.test8.g1.d1", + "root.test8.g1.d2", + "root.test8.g1.d3", + "root.test8.g1.d1", + "root.test8.g1.d2", + "root.test8.g1.d3", + "root.test8.g1.d1", + "root.test8.g1.d2", + "root.test8.g1.d3", + ], + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [ + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + ], + [ + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + ], + [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ], + ) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test8.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test8.g1.d2" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + with session.execute_query_statement( + "select * from root.test8.g1.d3" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test inserting multiple Record data belonging to the same non aligned device +def test_insert_records_of_one_device(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test9.g1") + ts_path_lst = [ + "root.test9.g1.d1.BOOLEAN", + "root.test9.g1.d1.INT32", + "root.test9.g1.d1.INT64", + "root.test9.g1.d1.FLOAT", + "root.test9.g1.d1.DOUBLE", + "root.test9.g1.d1.TEXT", + "root.test9.g1.d1.TS", + "root.test9.g1.d1.DATE", + "root.test9.g1.d1.BLOB", + "root.test9.g1.d1.STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_multi_time_series( + ts_path_lst, + data_type_lst, + encoding_lst, + compressor_lst, + props_lst=None, + tags_lst=None, + attributes_lst=None, + alias_lst=None, + ) + + expect = 9 + session.insert_records_of_one_device( + "root.test9.g1.d1", + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [ + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + ], + [ + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + ], + [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ], + ) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test9.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test inserting multiple Record data belonging to the same aligned device +def test_insert_aligned_records_of_one_device(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test10.g1") + measurements_lst = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_type_lst = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + encoding_lst = [ + TSEncoding.RLE, + TSEncoding.CHIMP, + TSEncoding.ZIGZAG, + TSEncoding.RLBE, + TSEncoding.SPRINTZ, + TSEncoding.DICTIONARY, + TSEncoding.TS_2DIFF, + TSEncoding.CHIMP, + TSEncoding.PLAIN, + TSEncoding.DICTIONARY, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + device11_id = "root.test10.g1.d1" + session.create_aligned_time_series( + device11_id, measurements_lst, data_type_lst, encoding_lst, compressor_lst + ) + expect = 9 + session.insert_aligned_records_of_one_device( + "root.test10.g1.d1", + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [ + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + ], + [ + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ], + ], + [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ], + ) + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test10.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test writes with type inference +def test_insert_str_record(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + # Preparation before testing + session.set_storage_group("root.test11.g1") + ts_path_lst = [ + "root.test9.g1.d1.BOOLEAN", + "root.test9.g1.d1.INT32", + "root.test9.g1.d1.INT64", + "root.test9.g1.d1.FLOAT", + "root.test9.g1.d1.DOUBLE", + "root.test9.g1.d1.TEXT", + "root.test9.g1.d1.TS", + "root.test9.g1.d1.DATE", + "root.test9.g1.d1.BLOB", + "root.test9.g1.d1.STRING", + ] + data_type_lst = [ + TSDataType.TEXT, + TSDataType.DOUBLE, + TSDataType.DOUBLE, + TSDataType.DOUBLE, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + ] + encoding_lst = [ + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + TSEncoding.PLAIN, + ] + compressor_lst = [ + Compressor.UNCOMPRESSED, + Compressor.SNAPPY, + Compressor.LZ4, + Compressor.GZIP, + Compressor.ZSTD, + Compressor.LZMA2, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + Compressor.GZIP, + ] + session.create_multi_time_series( + ts_path_lst, + data_type_lst, + encoding_lst, + compressor_lst, + props_lst=None, + tags_lst=None, + attributes_lst=None, + alias_lst=None, + ) + + expect = 1 + session.insert_str_record( + "root.test11.g1.d1", + 1, + [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ], + [ + "true", + "10", + "11", + "11.11", + "10011.1", + "test01", + "11", + "1970-01-01", + 'b"x12x34"', + "string01", + ], + ) + + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement( + "select * from root.test11.g1.d1" + ) as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() + + +# Test automatic create +def test_insert_auto_create(): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open() + + expect = 360 + # Test non aligned tablet writing + num = 0 + for i in range(1, 10): + device_id = "root.repeat.g1.fd_a" + str(num) + num = num + 1 + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_ = [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ] + timestamps_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + tablet_ = Tablet( + device_id, measurements_, data_types_, values_, timestamps_ + ) + session.insert_tablet(tablet_) + + # Test non aligned numpy tablet writing + num = 0 + for i in range(1, 10): + device_id = "root.repeat.g1.fd_b" + str(num) + num = num + 1 + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + np_tablet_ = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + session.insert_tablet(np_tablet_) + + # Test aligning tablet writing + num = 0 + for i in range(1, 10): + device_id = "root.repeat.g1.d_c" + str(num) + num = num + 1 + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_ = [ + [ + False, + 0, + 0, + 0.0, + 0.0, + "1234567890", + 0, + date(1970, 1, 1), + "1234567890".encode("utf-8"), + "1234567890", + ], + [ + True, + -2147483648, + -9223372036854775808, + -0.12345678, + -0.12345678901234567, + "abcdefghijklmnopqrstuvwsyz", + -9223372036854775808, + date(1000, 1, 1), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz", + ], + [ + True, + 2147483647, + 9223372036854775807, + 0.123456789, + 0.12345678901234567, + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + 9223372036854775807, + date(9999, 12, 31), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~", + ], + [ + True, + 1, + 1, + 1.0, + 1.0, + "没问题", + 1, + date(1970, 1, 1), + "没问题".encode("utf-8"), + "没问题", + ], + [ + True, + -1, + -1, + 1.1234567, + 1.1234567890123456, + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + 11, + date(1970, 1, 1), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + ], + [ + True, + 10, + 11, + 4.123456, + 4.123456789012345, + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + 11, + date(1970, 1, 1), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + ], + [ + True, + -10, + -11, + 12.12345, + 12.12345678901234, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + [ + None, + None, + None, + None, + None, + "", + None, + date(1970, 1, 1), + "".encode("utf-8"), + "", + ], + [ + True, + -0, + -0, + -0.0, + -0.0, + " ", + 11, + date(1970, 1, 1), + " ".encode("utf-8"), + " ", + ], + [ + True, + 10, + 11, + 1.1, + 10011.1, + "test01", + 11, + date(1970, 1, 1), + "Hello, World!".encode("utf-8"), + "string01", + ], + ] + timestamps_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + tablet_ = Tablet( + device_id, measurements_, data_types_, values_, timestamps_ + ) + session.insert_aligned_tablet(tablet_) + + # Test alignment of numpy tablet writing + num = 0 + for i in range(1, 10): + device_id = "root.repeat.g1.d_d" + str(num) + num = num + 1 + measurements_ = [ + "BOOLEAN", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "TEXT", + "TS", + "DATE", + "BLOB", + "STRING", + ] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, + ] + np_values_ = [ + np.array( + [False, True, False, True, False, True, False, True, False, True], + TSDataType.BOOLEAN.np_dtype(), + ), + np.array( + [0, -2147483648, 2147483647, 1, -1, 10, -10, -0, 10, 10], + TSDataType.INT32.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.INT64.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678, + 0.123456789, + 1.0, + 1.1234567, + 4.123456, + 12.12345, + -0.0, + 1.1, + 1.1, + ], + TSDataType.FLOAT.np_dtype(), + ), + np.array( + [ + 0.0, + -0.12345678901234567, + 0.12345678901234567, + 1.0, + 1.1234567890123456, + 4.123456789012345, + 12.12345678901234, + -0.0, + 1.1, + 1.1, + ], + TSDataType.DOUBLE.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.TEXT.np_dtype(), + ), + np.array( + [ + 0, + -9223372036854775808, + 9223372036854775807, + 1, + -1, + 10, + -10, + -0, + 10, + 10, + ], + TSDataType.TIMESTAMP.np_dtype(), + ), + np.array( + [ + date(1970, 1, 1), + date(1000, 1, 1), + date(9999, 12, 31), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + date(1970, 1, 1), + ], + TSDataType.DATE.np_dtype(), + ), + np.array( + [ + "1234567890".encode("utf-8"), + "abcdefghijklmnopqrstuvwsyz".encode("utf-8"), + "!@#$%^&*()_+}{|:`~-=[];,./<>?~".encode("utf-8"), + "没问题".encode("utf-8"), + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/".encode("utf-8"), + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题".encode( + "utf-8" + ), + "".encode("utf-8"), + " ".encode("utf-8"), + "1234567890".encode("utf-8"), + "1234567890".encode("utf-8"), + ], + TSDataType.BLOB.np_dtype(), + ), + np.array( + [ + "1234567890", + "abcdefghijklmnopqrstuvwsyz", + "!@#$%^&*()_+}{|:'`~-=[];,./<>?~", + "没问题", + "！@#￥%……&*（）——|：“《》？·【】、；‘，。/", + "1234567890abcdefghijklmnopqrstuvwsyz!@#$%^&*()_+}{|:'`~-=[];,./<>?~！@#￥%……&*（）——|：“《》？·【】、；‘，。/没问题", + "", + " ", + "test01", + "test01", + ], + TSDataType.STRING.np_dtype(), + ), + ] + np_timestamps_ = np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], TSDataType.INT64.np_dtype() + ) + np_tablet_ = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_ + ) + session.insert_aligned_tablet(np_tablet_) + + # Calculate the number of rows in the actual time series + actual = 0 + with session.execute_query_statement("show timeseries") as session_data_set: + session_data_set.set_fetch_size(1024) + while session_data_set.has_next(): + print(session_data_set.next()) + actual = actual + 1 + # Determine whether it meets expectations + assert expect == actual + session.close() diff --git a/iotdb-client/isession/pom.xml b/iotdb-client/isession/pom.xml index ed15ef7d98feb..263c310f31252 100644 --- a/iotdb-client/isession/pom.xml +++ b/iotdb-client/isession/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT isession IoTDB: Client: isession @@ -32,7 +32,7 @@ org.apache.iotdb service-rpc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.tsfile @@ -47,12 +47,12 @@ org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java index 85a8c9c00d53f..09b1444b4fccf 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java @@ -31,6 +31,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.record.Tablet; @@ -64,6 +65,14 @@ void open( INodeSupplier nodeSupplier) throws IoTDBConnectionException; + void open( + boolean enableRPCCompression, + int connectionTimeoutInMs, + Map deviceIdToEndpoint, + Map tabletModelDeviceIdToEndpoint, + INodeSupplier nodeSupplier) + throws IoTDBConnectionException; + void close() throws IoTDBConnectionException; String getTimeZone(); diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java new file mode 100644 index 0000000000000..75a37f8384719 --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.isession; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.write.record.Tablet; + +/** + * This interface defines a session for interacting with IoTDB tables. It supports operations such + * as data insertion, executing queries, and closing the session. Implementations of this interface + * are expected to manage connections and ensure proper resource cleanup. + * + *

Each method may throw exceptions to indicate issues such as connection errors or execution + * failures. + * + *

Since this interface extends {@link AutoCloseable}, it is recommended to use + * try-with-resources to ensure the session is properly closed. + */ +public interface ITableSession extends AutoCloseable { + + /** + * Inserts a {@link Tablet} into the database. + * + * @param tablet the tablet containing time-series data to be inserted. + * @throws StatementExecutionException if an error occurs while executing the statement. + * @throws IoTDBConnectionException if there is an issue with the IoTDB connection. + */ + void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException; + + /** + * Executes a non-query SQL statement, such as a DDL or DML command. + * + * @param sql the SQL statement to execute. + * @throws IoTDBConnectionException if there is an issue with the IoTDB connection. + * @throws StatementExecutionException if an error occurs while executing the statement. + */ + void executeNonQueryStatement(String sql) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Executes a query SQL statement and returns the result set. + * + * @param sql the SQL query statement to execute. + * @return a {@link SessionDataSet} containing the query results. + * @throws StatementExecutionException if an error occurs while executing the statement. + * @throws IoTDBConnectionException if there is an issue with the IoTDB connection. + */ + SessionDataSet executeQueryStatement(String sql) + throws StatementExecutionException, IoTDBConnectionException; + + /** + * Executes a query SQL statement with a specified timeout and returns the result set. + * + * @param sql the SQL query statement to execute. + * @param timeoutInMs the timeout duration in milliseconds for the query execution. + * @return a {@link SessionDataSet} containing the query results. + * @throws StatementExecutionException if an error occurs while executing the statement. + * @throws IoTDBConnectionException if there is an issue with the IoTDB connection. + */ + SessionDataSet executeQueryStatement(String sql, long timeoutInMs) + throws StatementExecutionException, IoTDBConnectionException; + + /** + * Closes the session, releasing any held resources. + * + * @throws IoTDBConnectionException if there is an issue with closing the IoTDB connection. + */ + @Override + void close() throws IoTDBConnectionException; +} diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java index a095b5e018855..611014c615c3b 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java @@ -54,5 +54,7 @@ public class SessionConfig { public static final long RETRY_INTERVAL_IN_MS = 500; + public static final String SQL_DIALECT = "tree"; + private SessionConfig() {} } diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java index 4b3d837c07ed5..fa67708fed0c7 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java @@ -29,17 +29,17 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; import java.sql.Timestamp; +import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Map; -import static org.apache.iotdb.rpc.IoTDBRpcDataSet.START_INDEX; - public class SessionDataSet implements ISessionDataSet { private final IoTDBRpcDataSet ioTDBRpcDataSet; @@ -57,7 +57,10 @@ public SessionDataSet( List queryResult, boolean ignoreTimeStamp, boolean moreData, - ZoneId zoneId) { + ZoneId zoneId, + int timeFactor, + boolean tableModel, + List columnIndex2TsBlockColumnIndexList) { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( sql, @@ -74,41 +77,10 @@ public SessionDataSet( SessionConfig.DEFAULT_FETCH_SIZE, 0, zoneId, - RpcUtils.DEFAULT_TIME_FORMAT); - } - - @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters - public SessionDataSet( - String sql, - List columnNameList, - List columnTypeList, - Map columnNameIndex, - long queryId, - long statementId, - IClientRPCService.Iface client, - long sessionId, - List queryResult, - boolean ignoreTimeStamp, - long timeout, - boolean moreData, - ZoneId zoneId) { - this.ioTDBRpcDataSet = - new IoTDBRpcDataSet( - sql, - columnNameList, - columnTypeList, - columnNameIndex, - ignoreTimeStamp, - moreData, - queryId, - statementId, - client, - sessionId, - queryResult, - SessionConfig.DEFAULT_FETCH_SIZE, - timeout, - zoneId, - RpcUtils.DEFAULT_TIME_FORMAT); + RpcUtils.DEFAULT_TIME_FORMAT, + timeFactor, + tableModel, + columnIndex2TsBlockColumnIndexList); } @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters @@ -126,7 +98,10 @@ public SessionDataSet( long timeout, boolean moreData, int fetchSize, - ZoneId zoneId) { + ZoneId zoneId, + int timeFactor, + boolean tableModel, + List columnIndex2TsBlockColumnIndexList) { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( sql, @@ -143,30 +118,33 @@ public SessionDataSet( fetchSize, timeout, zoneId, - RpcUtils.DEFAULT_TIME_FORMAT); + RpcUtils.DEFAULT_TIME_FORMAT, + timeFactor, + tableModel, + columnIndex2TsBlockColumnIndexList); } public int getFetchSize() { - return ioTDBRpcDataSet.fetchSize; + return ioTDBRpcDataSet.getFetchSize(); } public void setFetchSize(int fetchSize) { - ioTDBRpcDataSet.fetchSize = fetchSize; + ioTDBRpcDataSet.setFetchSize(fetchSize); } @Override public List getColumnNames() { - return new ArrayList<>(ioTDBRpcDataSet.columnNameList); + return new ArrayList<>(ioTDBRpcDataSet.getColumnNameList()); } @Override public List getColumnTypes() { - return new ArrayList<>(ioTDBRpcDataSet.columnTypeList); + return new ArrayList<>(ioTDBRpcDataSet.getColumnTypeList()); } @Override public boolean hasNext() throws StatementExecutionException, IoTDBConnectionException { - if (ioTDBRpcDataSet.hasCachedRecord) { + if (ioTDBRpcDataSet.hasCachedRecord()) { return true; } else { return ioTDBRpcDataSet.next(); @@ -175,62 +153,54 @@ public boolean hasNext() throws StatementExecutionException, IoTDBConnectionExce private RowRecord constructRowRecordFromValueArray() throws StatementExecutionException { List outFields = new ArrayList<>(); - for (int i = 0; i < ioTDBRpcDataSet.columnSize; i++) { + for (int i = ioTDBRpcDataSet.getValueColumnStartIndex(); + i < ioTDBRpcDataSet.getColumnSize(); + i++) { Field field; - int index = i + 1; - int datasetColumnIndex = i + START_INDEX; - if (ioTDBRpcDataSet.ignoreTimeStamp) { - index--; - datasetColumnIndex--; - } - int loc = - ioTDBRpcDataSet.columnOrdinalMap.get(ioTDBRpcDataSet.columnNameList.get(index)) - - START_INDEX; + String columnName = ioTDBRpcDataSet.getColumnNameList().get(i); - if (!ioTDBRpcDataSet.isNull(datasetColumnIndex)) { - TSDataType dataType = ioTDBRpcDataSet.columnTypeDeduplicatedList.get(loc); + if (!ioTDBRpcDataSet.isNull(columnName)) { + TSDataType dataType = ioTDBRpcDataSet.getDataType(columnName); field = new Field(dataType); switch (dataType) { case BOOLEAN: - boolean booleanValue = ioTDBRpcDataSet.getBoolean(datasetColumnIndex); + boolean booleanValue = ioTDBRpcDataSet.getBoolean(columnName); field.setBoolV(booleanValue); break; case INT32: case DATE: - int intValue = ioTDBRpcDataSet.getInt(datasetColumnIndex); + int intValue = ioTDBRpcDataSet.getInt(columnName); field.setIntV(intValue); break; case INT64: case TIMESTAMP: - long longValue = ioTDBRpcDataSet.getLong(datasetColumnIndex); + long longValue = ioTDBRpcDataSet.getLong(columnName); field.setLongV(longValue); break; case FLOAT: - float floatValue = ioTDBRpcDataSet.getFloat(datasetColumnIndex); + float floatValue = ioTDBRpcDataSet.getFloat(columnName); field.setFloatV(floatValue); break; case DOUBLE: - double doubleValue = ioTDBRpcDataSet.getDouble(datasetColumnIndex); + double doubleValue = ioTDBRpcDataSet.getDouble(columnName); field.setDoubleV(doubleValue); break; case TEXT: case BLOB: case STRING: - field.setBinaryV(ioTDBRpcDataSet.getBinary(datasetColumnIndex)); + field.setBinaryV(ioTDBRpcDataSet.getBinary(columnName)); break; default: throw new UnSupportedDataTypeException( - String.format( - "Data type %s is not supported.", - ioTDBRpcDataSet.columnTypeDeduplicatedList.get(i))); + String.format("Data type %s is not supported.", dataType)); } } else { field = new Field(null); } outFields.add(field); } - return new RowRecord(ioTDBRpcDataSet.time, outFields); + return new RowRecord(ioTDBRpcDataSet.getCurrentRowTime(), outFields); } /** @@ -244,10 +214,10 @@ private RowRecord constructRowRecordFromValueArray() throws StatementExecutionEx */ @Override public RowRecord next() throws StatementExecutionException, IoTDBConnectionException { - if (!ioTDBRpcDataSet.hasCachedRecord && !hasNext()) { + if (!ioTDBRpcDataSet.hasCachedRecord() && !hasNext()) { return null; } - ioTDBRpcDataSet.hasCachedRecord = false; + ioTDBRpcDataSet.setHasCachedRecord(false); return constructRowRecordFromValueArray(); } @@ -347,16 +317,32 @@ public Timestamp getTimestamp(String columnName) throws StatementExecutionExcept return ioTDBRpcDataSet.getTimestamp(columnName); } + public LocalDate getDate(int columnIndex) throws StatementExecutionException { + return ioTDBRpcDataSet.getDate(columnIndex); + } + + public LocalDate getDate(String columnName) throws StatementExecutionException { + return ioTDBRpcDataSet.getDate(columnName); + } + + public Binary getBlob(int columnIndex) throws StatementExecutionException { + return ioTDBRpcDataSet.getBinary(columnIndex); + } + + public Binary getBlob(String columnName) throws StatementExecutionException { + return ioTDBRpcDataSet.getBinary(columnName); + } + public int findColumn(String columnName) { return ioTDBRpcDataSet.findColumn(columnName); } public List getColumnNameList() { - return ioTDBRpcDataSet.columnNameList; + return ioTDBRpcDataSet.getColumnNameList(); } public List getColumnTypeList() { - return ioTDBRpcDataSet.columnTypeList; + return ioTDBRpcDataSet.getColumnTypeList(); } } } diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/pool/ITableSessionPool.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/pool/ITableSessionPool.java new file mode 100644 index 0000000000000..c9b9f7957726b --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/pool/ITableSessionPool.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.isession.pool; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.rpc.IoTDBConnectionException; + +/** + * This interface defines a pool for managing {@link ITableSession} instances. It provides methods + * to acquire a session from the pool and to close the pool. + * + *

The implementation should handle the lifecycle of sessions, ensuring efficient reuse and + * proper cleanup of resources. + */ +public interface ITableSessionPool extends AutoCloseable { + + /** + * Acquires an {@link ITableSession} instance from the pool. + * + * @return an {@link ITableSession} instance for interacting with IoTDB. + * @throws IoTDBConnectionException if there is an issue obtaining a session from the pool. + */ + ITableSession getSession() throws IoTDBConnectionException; + + /** + * Closes the session pool, releasing any held resources. + * + *

Once the pool is closed, no further sessions can be acquired. + */ + void close(); +} diff --git a/iotdb-client/jdbc/pom.xml b/iotdb-client/jdbc/pom.xml index 34634e26d1d21..88dd6e1161f22 100644 --- a/iotdb-client/jdbc/pom.xml +++ b/iotdb-client/jdbc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-jdbc IoTDB: Client: Jdbc @@ -38,12 +38,12 @@ org.apache.iotdb service-rpc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.tsfile @@ -58,7 +58,7 @@ org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.thrift @@ -175,6 +175,9 @@ ${project.build.outputDirectory}/META-INF/MANIFEST.MF + + ${project.version} + @@ -209,6 +212,31 @@ + + maven-assembly-plugin + ${maven.assembly.version} + + + jar-with-dependencies + + + + true + + + + + + make-assembly + + + single + + + package + + + @@ -275,32 +303,5 @@ true - - get-jar-with-dependencies - - - - maven-assembly-plugin - ${maven.assembly.version} - - - jar-with-dependencies - - - - - make-assembly - - - single - - - package - - - - - - diff --git a/iotdb-client/jdbc/src/main/feature/feature.xml b/iotdb-client/jdbc/src/main/feature/feature.xml index 8f37dae5f8c1f..033fe46d47c6c 100644 --- a/iotdb-client/jdbc/src/main/feature/feature.xml +++ b/iotdb-client/jdbc/src/main/feature/feature.xml @@ -18,7 +18,7 @@ --> - +

Feature to install required Bundle to use IoTDB inside Karaf container
wrap scr diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java deleted file mode 100644 index 55d7b0bbf5526..0000000000000 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/AbstractIoTDBJDBCResultSet.java +++ /dev/null @@ -1,1220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.jdbc; - -import org.apache.iotdb.rpc.IoTDBJDBCDataSet; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.service.rpc.thrift.IClientRPCService; - -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.InputStream; -import java.io.Reader; -import java.math.BigDecimal; -import java.math.MathContext; -import java.net.URL; -import java.sql.Array; -import java.sql.Blob; -import java.sql.Clob; -import java.sql.Date; -import java.sql.NClob; -import java.sql.Ref; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowId; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; -import java.time.ZoneId; -import java.util.BitSet; -import java.util.Calendar; -import java.util.List; -import java.util.Map; - -public abstract class AbstractIoTDBJDBCResultSet implements ResultSet { - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractIoTDBJDBCResultSet.class); - - protected Statement statement; - protected SQLWarning warningChain = null; - protected List columnTypeList; - protected IoTDBJDBCDataSet ioTDBRpcDataSet; - protected IoTDBTracingInfo ioTDBRpcTracingInfo; - private boolean isRpcFetchResult = true; - private List sgColumns; - private BitSet aliasColumnMap; - - @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters - public AbstractIoTDBJDBCResultSet( - Statement statement, - List columnNameList, - List columnTypeList, - Map columnNameIndex, - boolean ignoreTimeStamp, - IClientRPCService.Iface client, - String sql, - long queryId, - long sessionId, - long timeout, - List sgColumns, - BitSet aliasColumnMap, - ZoneId zoneId) - throws SQLException { - this.ioTDBRpcDataSet = - new IoTDBJDBCDataSet( - sql, - columnNameList, - columnTypeList, - columnNameIndex, - ignoreTimeStamp, - queryId, - ((IoTDBStatement) statement).getStmtId(), - client, - sessionId, - null, - statement.getFetchSize(), - timeout, - sgColumns, - aliasColumnMap, - zoneId); - this.statement = statement; - this.columnTypeList = columnTypeList; - this.aliasColumnMap = aliasColumnMap; - } - - @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters - protected AbstractIoTDBJDBCResultSet( - Statement statement, - List columnNameList, - List columnTypeList, - Map columnNameIndex, - boolean ignoreTimeStamp, - IClientRPCService.Iface client, - String sql, - long queryId, - long sessionId, - long timeout, - boolean isRpcFetchResult, - ZoneId zoneId) - throws SQLException { - this.ioTDBRpcDataSet = - new IoTDBJDBCDataSet( - sql, - columnNameList, - columnTypeList, - columnNameIndex, - ignoreTimeStamp, - queryId, - ((IoTDBStatement) statement).getStmtId(), - client, - sessionId, - null, - statement.getFetchSize(), - timeout, - zoneId); - this.statement = statement; - this.columnTypeList = columnTypeList; - this.isRpcFetchResult = isRpcFetchResult; - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public T unwrap(Class iface) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean absolute(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void afterLast() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void beforeFirst() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void cancelRowUpdates() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void clearWarnings() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void close() throws SQLException { - try { - ioTDBRpcDataSet.close(); - } catch (StatementExecutionException e) { - throw new SQLException("Error occurs for close operation in server side because ", e); - } catch (TException e) { - throw new SQLException("Error occurs when connecting to server for close operation ", e); - } - } - - @Override - public void deleteRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public int findColumn(String columnName) { - return ioTDBRpcDataSet.findColumn(columnName); - } - - @Override - public boolean first() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Array getArray(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Array getArray(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public InputStream getAsciiStream(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public InputStream getAsciiStream(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - try { - return getBigDecimal(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public BigDecimal getBigDecimal(String columnName) throws SQLException { - String value = getValueByName(columnName); - if (value != null) { - return new BigDecimal(value); - } else { - return null; - } - } - - @Override - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - MathContext mc = new MathContext(scale); - return getBigDecimal(columnIndex).round(mc); - } - - @Override - public BigDecimal getBigDecimal(String columnName, int scale) throws SQLException { - return getBigDecimal(findColumn(columnName), scale); - } - - @Override - public InputStream getBinaryStream(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public InputStream getBinaryStream(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Blob getBlob(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Blob getBlob(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean getBoolean(int columnIndex) throws SQLException { - try { - return getBoolean(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public boolean getBoolean(String columnName) throws SQLException { - try { - return ioTDBRpcDataSet.getBoolean(columnName); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public byte getByte(int columnIndex) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public byte getByte(String columnName) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public byte[] getBytes(int columnIndex) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public byte[] getBytes(String columnName) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Reader getCharacterStream(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Reader getCharacterStream(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Clob getClob(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Clob getClob(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public int getConcurrency() { - return ResultSet.CONCUR_READ_ONLY; - } - - @Override - public String getCursorName() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Date getDate(int columnIndex) throws SQLException { - return new Date(getLong(columnIndex)); - } - - @Override - public Date getDate(String columnName) throws SQLException { - return getDate(findColumn(columnName)); - } - - @Override - public Date getDate(int arg0, Calendar arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Date getDate(String arg0, Calendar arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public double getDouble(int columnIndex) throws SQLException { - try { - return getDouble(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public double getDouble(String columnName) throws SQLException { - try { - return ioTDBRpcDataSet.getDouble(columnName); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public int getFetchDirection() { - return ResultSet.FETCH_FORWARD; - } - - @Override - public void setFetchDirection(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public int getFetchSize() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void setFetchSize(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public float getFloat(int columnIndex) throws SQLException { - try { - return getFloat(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public float getFloat(String columnName) throws SQLException { - try { - return ioTDBRpcDataSet.getFloat(columnName); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public int getHoldability() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public int getInt(int columnIndex) throws SQLException { - try { - return getInt(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public int getInt(String columnName) throws SQLException { - try { - return ioTDBRpcDataSet.getInt(columnName); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public long getLong(int columnIndex) throws SQLException { - try { - return getLong(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public ResultSetMetaData getMetaData() { - String operationType = ""; - boolean nonAlign = false; - try { - if (statement.getResultSet() instanceof IoTDBJDBCResultSet) { - operationType = ((IoTDBJDBCResultSet) statement.getResultSet()).getOperationType(); - this.sgColumns = ((IoTDBJDBCResultSet) statement.getResultSet()).getSgColumns(); - } else if (statement.getResultSet() instanceof IoTDBNonAlignJDBCResultSet) { - operationType = ((IoTDBNonAlignJDBCResultSet) statement.getResultSet()).getOperationType(); - this.sgColumns = ((IoTDBNonAlignJDBCResultSet) statement.getResultSet()).getSgColumns(); - nonAlign = true; - } - } catch (SQLException throwables) { - LOGGER.error("Get meta data error: {}", throwables.getMessage()); - } - return new IoTDBResultMetadata( - nonAlign, - sgColumns, - operationType, - ioTDBRpcDataSet.columnNameList, - ioTDBRpcDataSet.columnTypeList, - ioTDBRpcDataSet.ignoreTimeStamp); - } - - @Override - public Reader getNCharacterStream(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Reader getNCharacterStream(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public NClob getNClob(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public NClob getNClob(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public String getNString(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public String getNString(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Object getObject(int columnIndex) throws SQLException { - try { - return getObject(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public Object getObject(String columnName) throws SQLException { - return getObjectByName(columnName); - } - - @Override - public Object getObject(int arg0, Map> arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Object getObject(String arg0, Map> arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public T getObject(int arg0, Class arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public T getObject(String arg0, Class arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Ref getRef(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Ref getRef(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public int getRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public RowId getRowId(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public RowId getRowId(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public SQLXML getSQLXML(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public SQLXML getSQLXML(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public short getShort(int columnIndex) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public short getShort(String columnName) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Statement getStatement() { - return this.statement; - } - - @Override - public String getString(int columnIndex) throws SQLException { - try { - return getString(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); - } catch (StatementExecutionException e) { - throw new SQLException(e.getMessage()); - } - } - - @Override - public String getString(String columnName) throws SQLException { - return getValueByName(columnName); - } - - @Override - public Time getTime(int columnIndex) throws SQLException { - return new Time(getLong(columnIndex)); - } - - @Override - public Time getTime(String columnName) throws SQLException { - return getTime(findColumn(columnName)); - } - - @Override - public Time getTime(int arg0, Calendar arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Time getTime(String arg0, Calendar arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Timestamp getTimestamp(int columnIndex) throws SQLException { - return new Timestamp(getLong(columnIndex)); - } - - @Override - public Timestamp getTimestamp(String columnName) throws SQLException { - return getTimestamp(findColumn(columnName)); - } - - @Override - public Timestamp getTimestamp(int arg0, Calendar arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public Timestamp getTimestamp(String arg0, Calendar arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public int getType() { - return ResultSet.TYPE_FORWARD_ONLY; - } - - @Override - public URL getURL(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public URL getURL(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public InputStream getUnicodeStream(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public InputStream getUnicodeStream(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public SQLWarning getWarnings() { - return warningChain; - } - - @Override - public void insertRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean isAfterLast() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean isBeforeFirst() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean isClosed() { - return ioTDBRpcDataSet.isClosed; - } - - @Override - public boolean isFirst() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean isLast() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean last() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void moveToCurrentRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void moveToInsertRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean next() throws SQLException { - if (hasCachedResults()) { - constructOneRow(); - return true; - } - if (ioTDBRpcDataSet.emptyResultSet) { - return false; - } - if (isRpcFetchResult && fetchResults()) { - constructOneRow(); - return true; - } - return false; - } - - /** - * @return true means has results - */ - abstract boolean fetchResults() throws SQLException; - - abstract boolean hasCachedResults(); - - abstract void constructOneRow(); - - @Override - public boolean previous() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void refreshRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean relative(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean rowDeleted() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean rowInserted() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean rowUpdated() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateArray(int arg0, Array arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateArray(String arg0, Array arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateAsciiStream(int arg0, InputStream arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateAsciiStream(String arg0, InputStream arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateAsciiStream(int arg0, InputStream arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateAsciiStream(String arg0, InputStream arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateAsciiStream(int arg0, InputStream arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateAsciiStream(String arg0, InputStream arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBigDecimal(int arg0, BigDecimal arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBigDecimal(String arg0, BigDecimal arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBinaryStream(int arg0, InputStream arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBinaryStream(String arg0, InputStream arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBinaryStream(int arg0, InputStream arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBinaryStream(String arg0, InputStream arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBinaryStream(int arg0, InputStream arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBinaryStream(String arg0, InputStream arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBlob(int arg0, Blob arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBlob(String arg0, Blob arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBlob(int arg0, InputStream arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBlob(String arg0, InputStream arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBlob(int arg0, InputStream arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBlob(String arg0, InputStream arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBoolean(int arg0, boolean arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBoolean(String arg0, boolean arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateByte(int arg0, byte arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateByte(String arg0, byte arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBytes(int arg0, byte[] arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateBytes(String arg0, byte[] arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateCharacterStream(int arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateCharacterStream(String arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateCharacterStream(int arg0, Reader arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateCharacterStream(String arg0, Reader arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateCharacterStream(int arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateCharacterStream(String arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateClob(int arg0, Clob arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateClob(String arg0, Clob arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateClob(int arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateClob(String arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateClob(int arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateClob(String arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateDate(int arg0, Date arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateDate(String arg0, Date arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateDouble(int arg0, double arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateDouble(String arg0, double arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateFloat(int arg0, float arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateFloat(String arg0, float arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateInt(int arg0, int arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateInt(String arg0, int arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateLong(int arg0, long arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateLong(String arg0, long arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNCharacterStream(int arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNCharacterStream(String arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNCharacterStream(int arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNCharacterStream(String arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNClob(int arg0, NClob arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNClob(String arg0, NClob arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNClob(int arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNClob(String arg0, Reader arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNClob(int arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNClob(String arg0, Reader arg1, long arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNString(int arg0, String arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNString(String arg0, String arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNull(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateNull(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateObject(int arg0, Object arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateObject(String arg0, Object arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateObject(int arg0, Object arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateObject(String arg0, Object arg1, int arg2) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateRef(int arg0, Ref arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateRef(String arg0, Ref arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateRow() throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateRowId(int arg0, RowId arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateRowId(String arg0, RowId arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateSQLXML(int arg0, SQLXML arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateSQLXML(String arg0, SQLXML arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateShort(int arg0, short arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateShort(String arg0, short arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateString(int arg0, String arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateString(String arg0, String arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateTime(int arg0, Time arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateTime(String arg0, Time arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateTimestamp(int arg0, Timestamp arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public void updateTimestamp(String arg0, Timestamp arg1) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); - } - - @Override - public boolean wasNull() { - return ioTDBRpcDataSet.lastReadWasNull; - } - - abstract void checkRecord() throws SQLException; - - abstract String getValueByName(String columnName) throws SQLException; - - abstract Object getObjectByName(String columnName) throws SQLException; - - public boolean isSetTracingInfo() { - if (ioTDBRpcTracingInfo == null) { - return false; - } - return ioTDBRpcTracingInfo.isSetTracingInfo(); - } - - public List getActivityList() { - return ioTDBRpcTracingInfo.getActivityList(); - } - - public List getElapsedTimeList() { - return ioTDBRpcTracingInfo.getElapsedTimeList(); - } - - public long getStatisticsByName(String name) throws Exception { - return ioTDBRpcTracingInfo.getStatisticsByName(name); - } - - public String getStatisticsInfoByName(String name) throws Exception { - return ioTDBRpcTracingInfo.getStatisticsInfoByName(name); - } -} diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java index 4e2e0f38300f6..0b1049330eae7 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java @@ -81,4 +81,8 @@ private Config() { public static final String TRUST_STORE = "trust_store"; public static final String TRUST_STORE_PWD = "trust_store_pwd"; + + public static final String SQL_DIALECT = "sql_dialect"; + + public static final String DATABASE = "db"; } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java index 66b1e57d092a8..ecb0b552a81d9 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java @@ -24,8 +24,10 @@ public class Constant { private Constant() {} public static final String GLOBAL_DB_NAME = "IoTDB"; + public static final String TABLE_DIALECT = "table"; + public static final String TREE_DIALECT = "tree"; - static final String METHOD_NOT_SUPPORTED = "Method not supported"; + public static final String METHOD_NOT_SUPPORTED = "Method not supported"; static final String PARAMETER_NOT_NULL = "The parameter cannot be null"; static final String PARAMETER_SUPPORTED = "Parameter only supports BOOLEAN,INT32,INT64,FLOAT,DOUBLE,TEXT data type"; @@ -42,6 +44,9 @@ private Constant() {} public static final String STATISTICS_RESULT_LINES = "* Lines of result: %d"; public static final String STATISTICS_PRC_INFO = "* Num of RPC: %d, avg cost: %d ms"; + public static final String TREE = "tree"; + public static final String TABLE = "table"; + // version number public enum Version { V_0_12, diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java index 6ec6a9b5baf10..d1d830fb2c2cd 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/GroupedLSBWatermarkEncoder.java @@ -134,6 +134,10 @@ public RowRecord encodeRecord(RowRecord rowRecord) { double originDoubleValue = field.getDoubleV(); field.setDoubleV(encodeDouble(originDoubleValue, timestamp)); break; + case BLOB: + case STRING: + case BOOLEAN: + case TEXT: default: } } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java new file mode 100644 index 0000000000000..18069bf6faa1a --- /dev/null +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java @@ -0,0 +1,2935 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.jdbc; + +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; + +import org.apache.thrift.TException; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.utils.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.RowIdLifetime; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public abstract class IoTDBAbstractDatabaseMetadata implements DatabaseMetaData { + + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBAbstractDatabaseMetadata.class); + private static final String METHOD_NOT_SUPPORTED_STRING = "Method not supported"; + protected static final String CONVERT_ERROR_MSG = "Convert tsBlock error: {}"; + + protected IoTDBConnection connection; + protected IClientRPCService.Iface client; + protected long sessionId; + protected ZoneId zoneId; + + protected static final String BOOLEAN = "BOOLEAN"; + protected static final String DOUBLE = "DOUBLE"; + protected static final String FLOAT = "FLOAT"; + protected static final String INT32 = "INT32"; + protected static final String INT64 = "INT64"; + protected static final String STRING = "STRING"; + protected static final String BLOB = "BLOB"; + protected static final String TIMESTAMP = "TIMESTAMP"; + protected static final String DATE = "DATE"; + + protected static String sqlKeywordsThatArentSQL92; + + protected static final String BUFFER_LENGTH = "BUFFER_LENGTH"; + protected static final String CHAR_OCTET_LENGTH = "CHAR_OCTET_LENGTH"; + protected static final String COLUMN_NAME = "COLUMN_NAME"; + protected static final String COLUMN_SIZE = "COLUMN_SIZE"; + private static final String COLUMN_TYPE = "COLUMN_TYPE"; + protected static final String DATA_TYPE = "DATA_TYPE"; + protected static final String DECIMAL_DIGITS = "DECIMAL_DIGITS"; + private static final String DEFERRABILITY = "DEFERRABILITY"; + private static final String DELETE_RULE = "DELETE_RULE"; + private static final String FK_NAME = "FK_NAME"; + private static final String FKCOLUMN_NAME = "FKCOLUMN_NAME"; + private static final String FKTABLE_CAT = "FKTABLE_CAT"; + private static final String FKTABLE_NAME = "FKTABLE_NAME"; + private static final String FKTABLE_SCHEM = "FKTABLE_SCHEM"; + private static final String FUNCTION_CAT = "FUNCTION_CAT"; + private static final String FUNCTION_NAME = "FUNCTION_NAME"; + private static final String FUNCTION_SCHEM = "FUNCTION_SCHEM"; + private static final String FUNCTION_TYPE = "FUNCTION_TYPE"; + private static final String GRANTEE = "GRANTEE"; + private static final String GRANTOR = "GRANTOR"; + protected static final String IS_AUTOINCREMENT = "IS_AUTOINCREMENT"; + private static final String IS_GRANTABLE = "IS_GRANTABLE"; + protected static final String IS_NULLABLE = "IS_NULLABLE"; + protected static final String KEY_SEQ = "KEY_SEQ"; + private static final String LENGTH = "LENGTH"; + protected static final String NULLABLE = "NULLABLE"; + protected static final String NUM_PREC_RADIX = "NUM_PREC_RADIX"; + protected static final String ORDINAL_POSITION = "ORDINAL_POSITION"; + protected static final String PK_NAME = "PK_NAME"; + private static final String PKCOLUMN_NAME = "PKCOLUMN_NAME"; + private static final String PKTABLE_CAT = "PKTABLE_CAT"; + private static final String PKTABLE_NAME = "PKTABLE_NAME"; + private static final String PKTABLE_SCHEM = "PKTABLE_SCHEM"; + protected static final String PRECISION = "PRECISION"; + protected static final String PRIMARY = "PRIMARY"; + private static final String PRIVILEGE = "PRIVILEGE"; + private static final String PROCEDURE_CAT = "PROCEDURE_CAT"; + private static final String PROCEDURE_NAME = "PROCEDURE_NAME"; + private static final String PROCEDURE_SCHEM = "PROCEDURE_SCHEM"; + private static final String PROCEDURE_TYPE = "PROCEDURE_TYPE"; + private static final String PSEUDO_COLUMN = "PSEUDO_COLUMN"; + private static final String RADIX = "RADIX"; + protected static final String REMARKS = "REMARKS"; + protected static final String COLUMN_DEF = "COLUMN_DEF"; + protected static final String SCOPE_CATALOG = "SCOPE_CATALOG"; + protected static final String SCOPE_SCHEMA = "SCOPE_SCHEMA"; + protected static final String SCOPE_TABLE = "SCOPE_TABLE"; + protected static final String SOURCE_DATA_TYPE = "SOURCE_DATA_TYPE"; + protected static final String IS_GENERATEDCOLUMN = "IS_GENERATEDCOLUMN"; + private static final String SCALE = "SCALE"; + private static final String SCOPE = "SCOPE"; + private static final String SPECIFIC_NAME = "SPECIFIC_NAME"; + protected static final String SQL_DATA_TYPE = "SQL_DATA_TYPE"; + protected static final String SQL_DATETIME_SUB = "SQL_DATETIME_SUB"; + protected static final String TABLE_CAT = "TABLE_CAT"; + protected static final String TABLE_SCHEM = "TABLE_SCHEM"; + protected static final String TABLE_NAME = "TABLE_NAME"; + protected static final String TABLE_TYPE = "TABLE_TYPE"; + protected static final String TYPE_CAT = "TYPE_CAT"; + protected static final String TYPE_NAME = "TYPE_NAME"; + protected static final String TYPE_SCHEM = "TYPE_SCHEM"; + private static final String UPDATE_RULE = "UPDATE_RULE"; + + private static final String SHOW_FUNCTIONS = "show functions"; + protected static final String SHOW_DATABASES_SQL = "SHOW DATABASES "; + + private static TsBlockSerde serde = new TsBlockSerde(); + + public IoTDBAbstractDatabaseMetadata( + IoTDBConnection connection, IClientRPCService.Iface client, long sessionId, ZoneId zoneId) { + this.connection = connection; + this.client = client; + this.sessionId = sessionId; + this.zoneId = zoneId; + } + + protected static String[] sql92Keywords = { + "ABSOLUTE", + "EXEC", + "OVERLAPS", + "ACTION", + "EXECUTE", + "PAD", + "ADA", + "EXISTS", + "PARTIAL", + "ADD", + "EXTERNAL", + "PASCAL", + "ALL", + "EXTRACT", + "POSITION", + "ALLOCATE", + "FALSE", + PRECISION, + "ALTER", + "FETCH", + "PREPARE", + "AND", + "FIRST", + "PRESERVE", + "ANY", + FLOAT, + PRIMARY, + "ARE", + "FOR", + "PRIOR", + "AS", + "FOREIGN", + "PRIVILEGES", + "ASC", + "FORTRAN", + "PROCEDURE", + "ASSERTION", + "FOUND", + "PUBLIC", + "AT", + "FROM", + "READ", + "AUTHORIZATION", + "FULL", + "REAL", + "AVG", + "GET", + "REFERENCES", + "BEGIN", + "GLOBAL", + "RELATIVE", + "BETWEEN", + "GO", + "RESTRICT", + "BIT", + "GOTO", + "REVOKE", + "BIT_LENGTH", + "GRANT", + "RIGHT", + "BOTH", + "GROUP", + "ROLLBACK", + "BY", + "HAVING", + "ROWS", + "CASCADE", + "HOUR", + "SCHEMA", + "CASCADED", + "IDENTITY", + "SCROLL", + "CASE", + "IMMEDIATE", + "SECOND", + "CAST", + "IN", + "SECTION", + "CATALOG", + "INCLUDE", + "SELECT", + "CHAR", + "INDEX", + "SESSION", + "CHAR_LENGTH", + "INDICATOR", + "SESSION_USER", + "CHARACTER", + "INITIALLY", + "SET", + "CHARACTER_LENGTH", + "INNER", + "SIZE", + "CHECK", + "INPUT", + "SMALLINT", + "CLOSE", + "INSENSITIVE", + "SOME", + "COALESCE", + "INSERT", + "SPACE", + "COLLATE", + "INT", + "SQL", + "COLLATION", + "INTEGER", + "SQLCA", + "COLUMN", + "INTERSECT", + "SQLCODE", + "COMMIT", + "INTERVAL", + "SQLERROR", + "CONNECT", + "INTO", + "SQLSTATE", + "CONNECTION", + "IS", + "SQLWARNING", + "CONSTRAINT", + "ISOLATION", + "SUBSTRING", + "CONSTRAINTS", + "JOIN", + "SUM", + "CONTINUE", + "KEY", + "SYSTEM_USER", + "CONVERT", + "LANGUAGE", + "TABLE", + "CORRESPONDING", + "LAST", + "TEMPORARY", + "COUNT", + "LEADING", + "THEN", + "CREATE", + "LEFT", + "TIME", + "CROSS", + "LEVEL", + "TIMESTAMP", + "CURRENT", + "LIKE", + "TIMEZONE_HOUR", + "CURRENT_DATE", + "LOCAL", + "TIMEZONE_MINUTE", + "CURRENT_TIME", + "LOWER", + "TO", + "CURRENT_TIMESTAMP", + "MATCH", + "TRAILING", + "CURRENT_USER", + "MAX", + "TRANSACTION", + "CURSOR", + "MIN", + "TRANSLATE", + "DATE", + "MINUTE", + "TRANSLATION", + "DAY", + "MODULE", + "TRIM", + "DEALLOCATE", + "MONTH", + "TRUE", + "DEC", + "NAMES", + "UNION", + "DECIMAL", + "NATIONAL", + "UNIQUE", + "DECLARE", + "NATURAL", + "UNKNOWN", + "DEFAULT", + "NCHAR", + "UPDATE", + "DEFERRABLE", + "NEXT", + "UPPER", + "DEFERRED", + "NO", + "USAGE", + "DELETE", + "NONE", + "USER", + "DESC", + "NOT", + "USING", + "DESCRIBE", + "NULL", + "VALUE", + "DESCRIPTOR", + "NULLIF", + "VALUES", + "DIAGNOSTICS", + "NUMERIC", + "VARCHAR", + "DISCONNECT", + "OCTET_LENGTH", + "VARYING", + "DISTINCT", + "OF", + "VIEW", + "DOMAIN", + "ON", + "WHEN", + DOUBLE, + "ONLY", + "WHENEVER", + "DROP", + "OPEN", + "WHERE", + "ELSE", + "OPTION", + "WITH", + "END", + "OR", + "WORK", + "END-EXEC", + "ORDER", + "WRITE", + "ESCAPE", + "OUTER", + "YEAR", + "EXCEPT", + "OUTPUT", + "ZONE", + "EXCEPTION" + }; + + public static ByteBuffer convertTsBlock( + List> valuesList, List tsDataTypeList) throws IOException { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(tsDataTypeList); + for (List valuesInRow : valuesList) { + tsBlockBuilder.getTimeColumnBuilder().writeLong(0); + for (int j = 0; j < tsDataTypeList.size(); j++) { + TSDataType columnType = tsDataTypeList.get(j); + switch (columnType) { + case TEXT: + case STRING: + case BLOB: + tsBlockBuilder + .getColumnBuilder(j) + .writeBinary( + new Binary(valuesInRow.get(j).toString(), TSFileConfig.STRING_CHARSET)); + break; + case FLOAT: + tsBlockBuilder.getColumnBuilder(j).writeFloat((float) valuesInRow.get(j)); + break; + case INT32: + case DATE: + tsBlockBuilder.getColumnBuilder(j).writeInt((int) valuesInRow.get(j)); + break; + case INT64: + case TIMESTAMP: + tsBlockBuilder.getColumnBuilder(j).writeLong((long) valuesInRow.get(j)); + break; + case DOUBLE: + tsBlockBuilder.getColumnBuilder(j).writeDouble((double) valuesInRow.get(j)); + break; + case BOOLEAN: + tsBlockBuilder.getColumnBuilder(j).writeBoolean((boolean) valuesInRow.get(j)); + break; + default: + LOGGER.error("No data type was matched: {}", columnType); + break; + } + } + tsBlockBuilder.declarePosition(); + } + TsBlock tsBlock = tsBlockBuilder.build(); + if (tsBlock == null) { + return null; + } else { + return serde.serialize(tsBlock); + } + } + + protected void close(ResultSet rs, Statement stmt) { + try { + if (rs != null) { + rs.close(); + } + } catch (Exception ex) { + rs = null; + } + try { + if (stmt != null) { + stmt.close(); + } + } catch (Exception ex) { + stmt = null; + } + } + + protected int getSQLType(String columnType) { + switch (columnType.toUpperCase()) { + case BOOLEAN: + return Types.BOOLEAN; + case INT32: + return Types.INTEGER; + case INT64: + return Types.BIGINT; + case FLOAT: + return Types.FLOAT; + case DOUBLE: + return Types.DOUBLE; + case "TEXT": + return Types.LONGVARCHAR; + case STRING: + return Types.VARCHAR; + case BLOB: + return Types.BLOB; + case TIMESTAMP: + return Types.TIMESTAMP; + case DATE: + return Types.DATE; + default: + break; + } + return 0; + } + + protected int getTypePrecision(String columnType) { + switch (columnType.toUpperCase()) { + case BOOLEAN: + return 1; + case INT32: + return 10; + case INT64: + return 19; + case FLOAT: + return 38; + case DOUBLE: + return 308; + case "TEXT": + case BLOB: + case STRING: + return Integer.MAX_VALUE; + case TIMESTAMP: + return 3; + case DATE: + return 0; + default: + break; + } + return 0; + } + + protected int getTypeScale(String columnType) { + switch (columnType.toUpperCase()) { + case BOOLEAN: + case INT32: + case INT64: + case "TEXT": + case BLOB: + case STRING: + case TIMESTAMP: + case DATE: + return 0; + case FLOAT: + return 6; + case DOUBLE: + return 15; + default: + break; + } + return 0; + } + + @Override + public boolean allProceduresAreCallable() throws SQLException { + return false; + } + + @Override + public boolean allTablesAreSelectable() throws SQLException { + return true; + } + + @Override + public String getURL() throws SQLException { + // TODO: Return the URL for this DBMS or null if it cannot be generated + return this.connection.getUrl(); + } + + @Override + public String getUserName() throws SQLException { + return connection.getUserName(); + } + + @Override + public boolean isReadOnly() throws SQLException { + try { + return client.getProperties().isReadOnly; + } catch (TException e) { + LOGGER.error("Get is readOnly error: {}", e.getMessage()); + } + throw new SQLException("Can not get the read-only mode"); + } + + @Override + public boolean nullsAreSortedHigh() throws SQLException { + return false; + } + + @Override + public boolean nullsAreSortedLow() throws SQLException { + return false; + } + + @Override + public boolean nullsAreSortedAtStart() throws SQLException { + return false; + } + + @Override + public boolean nullsAreSortedAtEnd() throws SQLException { + return false; + } + + @Override + public String getDatabaseProductName() throws SQLException { + return Constant.GLOBAL_DB_NAME; + } + + @Override + public String getDatabaseProductVersion() throws SQLException { + String serverVersion = ""; + Statement stmt = this.connection.createStatement(); + ResultSet rs; + try { + String sql = "SHOW VERSION"; + rs = stmt.executeQuery(sql); + } catch (SQLException e) { + stmt.close(); + LOGGER.error("Get database product version error: {}", e.getMessage()); + throw e; + } + while (rs.next()) { + serverVersion = rs.getString("Version"); + } + + return serverVersion; + } + + @Override + public String getDriverName() throws SQLException { + return org.apache.iotdb.jdbc.IoTDBDriver.class.getName(); + } + + @Override + public abstract String getDriverVersion() throws SQLException; + + @Override + public int getDriverMajorVersion() { + return 4; + } + + @Override + public int getDriverMinorVersion() { + return 3; + } + + @Override + public boolean usesLocalFiles() throws SQLException { + return false; + } + + @Override + public boolean usesLocalFilePerTable() throws SQLException { + return false; + } + + @Override + public boolean supportsMixedCaseIdentifiers() throws SQLException { + return true; + } + + @Override + public boolean storesUpperCaseIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesLowerCaseIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesMixedCaseIdentifiers() throws SQLException { + return true; + } + + @Override + public boolean supportsMixedCaseQuotedIdentifiers() throws SQLException { + return true; + } + + @Override + public boolean storesUpperCaseQuotedIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesLowerCaseQuotedIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesMixedCaseQuotedIdentifiers() throws SQLException { + return true; + } + + @Override + public abstract String getIdentifierQuoteString() throws SQLException; + + @Override + public String getSQLKeywords() throws SQLException { + return sqlKeywordsThatArentSQL92; + } + + public abstract String getNumericFunctions() throws SQLException; + + @Override + public String getStringFunctions() throws SQLException { + return getSystemFunctions(); + } + + @Override + public String getSystemFunctions() throws SQLException { + String result = ""; + Statement statement = null; + ResultSet resultSet = null; + try { + statement = connection.createStatement(); + StringBuilder str = new StringBuilder(""); + resultSet = statement.executeQuery(SHOW_FUNCTIONS); + while (resultSet.next()) { + str.append(resultSet.getString(1)).append(","); + } + result = str.toString(); + if (!result.isEmpty()) { + result = result.substring(0, result.length() - 1); + } + } catch (Exception ex) { + LOGGER.error("Get system functions error: {}", ex.getMessage()); + } finally { + close(resultSet, statement); + } + return result; + } + + public abstract String getTimeDateFunctions() throws SQLException; + + @Override + public String getSearchStringEscape() throws SQLException { + return "\\"; + } + + @Override + public String getExtraNameCharacters() throws SQLException { + return ""; + } + + @Override + public boolean supportsAlterTableWithAddColumn() throws SQLException { + return true; + } + + @Override + public boolean supportsAlterTableWithDropColumn() throws SQLException { + return true; + } + + @Override + public boolean supportsColumnAliasing() throws SQLException { + return true; + } + + @Override + public boolean nullPlusNonNullIsNull() throws SQLException { + return false; + } + + @Override + public boolean supportsConvert() throws SQLException { + return false; + } + + @Override + public boolean supportsConvert(int fromType, int toType) throws SQLException { + return false; + } + + @Override + public boolean supportsTableCorrelationNames() throws SQLException { + return false; + } + + @Override + public boolean supportsDifferentTableCorrelationNames() throws SQLException { + return false; + } + + @Override + public boolean supportsExpressionsInOrderBy() throws SQLException { + return true; + } + + @Override + public boolean supportsOrderByUnrelated() throws SQLException { + return true; + } + + @Override + public boolean supportsGroupBy() throws SQLException { + return true; + } + + @Override + public boolean supportsGroupByUnrelated() throws SQLException { + return true; + } + + @Override + public boolean supportsGroupByBeyondSelect() throws SQLException { + return true; + } + + @Override + public boolean supportsLikeEscapeClause() throws SQLException { + return false; + } + + @Override + public boolean supportsMultipleResultSets() throws SQLException { + return false; + } + + @Override + public boolean supportsMultipleTransactions() throws SQLException { + return true; + } + + @Override + public boolean supportsNonNullableColumns() throws SQLException { + return false; + } + + @Override + public boolean supportsMinimumSQLGrammar() throws SQLException { + return false; + } + + @Override + public boolean supportsCoreSQLGrammar() throws SQLException { + return false; + } + + @Override + public boolean supportsExtendedSQLGrammar() throws SQLException { + return false; + } + + @Override + public boolean supportsANSI92EntryLevelSQL() throws SQLException { + return false; + } + + @Override + public boolean supportsANSI92IntermediateSQL() throws SQLException { + return false; + } + + @Override + public boolean supportsANSI92FullSQL() throws SQLException { + return false; + } + + @Override + public boolean supportsIntegrityEnhancementFacility() throws SQLException { + return false; + } + + @Override + public boolean supportsOuterJoins() throws SQLException { + return true; + } + + @Override + public boolean supportsFullOuterJoins() throws SQLException { + return true; + } + + @Override + public boolean supportsLimitedOuterJoins() throws SQLException { + return true; + } + + public abstract String getSchemaTerm() throws SQLException; + + @Override + public String getProcedureTerm() throws SQLException { + return ""; + } + + @Override + public String getCatalogTerm() throws SQLException { + return "database"; + } + + @Override + public boolean isCatalogAtStart() throws SQLException { + return false; + } + + @Override + public String getCatalogSeparator() throws SQLException { + return "."; + } + + @Override + public abstract boolean supportsSchemasInDataManipulation() throws SQLException; + + @Override + public boolean supportsSchemasInProcedureCalls() throws SQLException { + return false; + } + + @Override + public boolean supportsSchemasInTableDefinitions() throws SQLException { + return false; + } + + @Override + public boolean supportsSchemasInIndexDefinitions() throws SQLException { + return false; + } + + @Override + public boolean supportsSchemasInPrivilegeDefinitions() throws SQLException { + return true; + } + + @Override + public boolean supportsCatalogsInDataManipulation() throws SQLException { + return true; + } + + @Override + public boolean supportsCatalogsInProcedureCalls() throws SQLException { + return true; + } + + @Override + public boolean supportsCatalogsInTableDefinitions() throws SQLException { + return false; + } + + @Override + public boolean supportsCatalogsInIndexDefinitions() throws SQLException { + return true; + } + + @Override + public boolean supportsCatalogsInPrivilegeDefinitions() throws SQLException { + return true; + } + + @Override + public boolean supportsPositionedDelete() throws SQLException { + return false; + } + + @Override + public boolean supportsPositionedUpdate() throws SQLException { + return false; + } + + @Override + public boolean supportsSelectForUpdate() throws SQLException { + return false; + } + + @Override + public boolean supportsStoredProcedures() throws SQLException { + return false; + } + + @Override + public boolean supportsSubqueriesInComparisons() throws SQLException { + return false; + } + + @Override + public boolean supportsSubqueriesInExists() throws SQLException { + return false; + } + + @Override + public boolean supportsSubqueriesInIns() throws SQLException { + return false; + } + + @Override + public boolean supportsSubqueriesInQuantifieds() throws SQLException { + return false; + } + + @Override + public boolean supportsCorrelatedSubqueries() throws SQLException { + return false; + } + + @Override + public boolean supportsUnion() throws SQLException { + return false; + } + + @Override + public boolean supportsUnionAll() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenCursorsAcrossCommit() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenCursorsAcrossRollback() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenStatementsAcrossCommit() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenStatementsAcrossRollback() throws SQLException { + return false; + } + + @Override + public int getMaxBinaryLiteralLength() throws SQLException { + return Integer.MAX_VALUE; + } + + @Override + public int getMaxCharLiteralLength() throws SQLException { + return Integer.MAX_VALUE; + } + + @Override + public int getMaxColumnNameLength() throws SQLException { + return 1024; + } + + @Override + public int getMaxColumnsInGroupBy() throws SQLException { + return 1; + } + + @Override + public int getMaxColumnsInIndex() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInOrderBy() throws SQLException { + return 1; + } + + @Override + public int getMaxColumnsInSelect() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInTable() throws SQLException { + return Integer.MAX_VALUE; + } + + @Override + public int getMaxConnections() throws SQLException { + int maxcount = 0; + try { + maxcount = client.getProperties().getMaxConcurrentClientNum(); + } catch (TException e) { + LOGGER.error("Get max concurrentClientNUm error: {}", e.getMessage()); + } + return maxcount; + } + + @Override + public int getMaxCursorNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxIndexLength() throws SQLException { + return Integer.MAX_VALUE; + } + + @Override + public int getMaxSchemaNameLength() throws SQLException { + return 1024; + } + + @Override + public int getMaxProcedureNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxCatalogNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxRowSize() throws SQLException { + return 2147483639; + } + + @Override + public boolean doesMaxRowSizeIncludeBlobs() throws SQLException { + return false; + } + + @Override + public int getMaxStatementLength() throws SQLException { + try { + return client.getProperties().getThriftMaxFrameSize(); + } catch (TException e) { + LOGGER.error("Get max statement length error: {}", e.getMessage()); + } + return 0; + } + + @Override + public int getMaxStatements() throws SQLException { + return 0; + } + + @Override + public int getMaxTableNameLength() throws SQLException { + return 1024; + } + + @Override + public int getMaxTablesInSelect() throws SQLException { + return 1024; + } + + @Override + public int getMaxUserNameLength() throws SQLException { + return 1024; + } + + @Override + public int getDefaultTransactionIsolation() throws SQLException { + return 0; + } + + @Override + public boolean supportsTransactions() throws SQLException { + return false; + } + + @Override + public boolean supportsTransactionIsolationLevel(int level) throws SQLException { + return false; + } + + @Override + public boolean supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + return false; + } + + @Override + public boolean supportsDataManipulationTransactionsOnly() throws SQLException { + return true; + } + + @Override + public boolean dataDefinitionCausesTransactionCommit() throws SQLException { + return false; + } + + @Override + public boolean dataDefinitionIgnoredInTransactions() throws SQLException { + return false; + } + + @Override + public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[6]; + fields[0] = new Field("", PROCEDURE_CAT, "TEXT"); + fields[1] = new Field("", PROCEDURE_SCHEM, "TEXT"); + fields[2] = new Field("", PROCEDURE_NAME, "TEXT"); + fields[3] = new Field("", REMARKS, "TEXT"); + fields[4] = new Field("", PROCEDURE_TYPE, "TEXT"); + fields[5] = new Field("", SPECIFIC_NAME, "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get procedures error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getProcedureColumns( + String catalog, String schemaPattern, String procedureNamePattern, String columnNamePattern) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + + try { + Field[] fields = new Field[20]; + fields[0] = new Field("", PROCEDURE_CAT, "TEXT"); + fields[1] = new Field("", PROCEDURE_SCHEM, "TEXT"); + fields[2] = new Field("", PROCEDURE_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", COLUMN_TYPE, "TEXT"); + fields[5] = new Field("", DATA_TYPE, INT32); + fields[6] = new Field("", TYPE_NAME, "TEXT"); + fields[7] = new Field("", PRECISION, "TEXT"); + fields[8] = new Field("", LENGTH, "TEXT"); + fields[9] = new Field("", SCALE, "TEXT"); + fields[10] = new Field("", RADIX, "TEXT"); + fields[11] = new Field("", NULLABLE, "TEXT"); + fields[12] = new Field("", REMARKS, "TEXT"); + fields[13] = new Field("", "COLUMN_DEF", "TEXT"); + fields[14] = new Field("", SQL_DATA_TYPE, INT32); + fields[15] = new Field("", SQL_DATETIME_SUB, "TEXT"); + fields[16] = new Field("", CHAR_OCTET_LENGTH, "TEXT"); + fields[17] = new Field("", ORDINAL_POSITION, "TEXT"); + fields[18] = new Field("", IS_NULLABLE, "TEXT"); + fields[19] = new Field("", SPECIFIC_NAME, "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get procedure columns error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public abstract ResultSet getTables( + String catalog, String schemaPattern, String tableNamePattern, String[] types) + throws SQLException; + + @Override + public ResultSet getSchemas() throws SQLException { + Statement stmt = this.connection.createStatement(); + ResultSet rs; + try { + rs = stmt.executeQuery(SHOW_DATABASES_SQL); + } catch (SQLException e) { + stmt.close(); + throw e; + } + Field[] fields = new Field[2]; + fields[0] = new Field("", TABLE_SCHEM, "TEXT"); + fields[1] = new Field("", "TABLE_CATALOG", "TEXT"); + + List tsDataTypeList = Arrays.asList(TSDataType.TEXT, TSDataType.TEXT); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + while (rs.next()) { + String database = rs.getString(1); + List valueInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + valueInRow.add(database); + } + valuesList.add(valueInRow); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(rs, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getCatalogs() throws SQLException { + Statement stmt = this.connection.createStatement(); + ResultSet rs; + try { + rs = stmt.executeQuery(SHOW_DATABASES_SQL); + } catch (SQLException e) { + stmt.close(); + throw e; + } + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List tsDataTypeList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + tsDataTypeList.add(TSDataType.TEXT); + + while (rs.next()) { + List values = new ArrayList<>(); + values.add(rs.getString(1)); + valuesList.add(values); + } + columnNameList.add(TYPE_CAT); + columnTypeList.add("TEXT"); + columnNameIndex.put(TYPE_CAT, 0); + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error("Get catalogs error: {}", e.getMessage()); + } finally { + close(rs, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getTableTypes() throws SQLException { + Statement stmt = this.connection.createStatement(); + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List tsDataTypeList = new ArrayList<>(); + List value = new ArrayList<>(); + + tsDataTypeList.add(TSDataType.TEXT); + value.add("table"); + columnNameList.add(TABLE_TYPE); + columnTypeList.add("TEXT"); + columnNameIndex.put(TABLE_TYPE, 0); + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(Collections.singletonList(value), tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public abstract ResultSet getColumns( + String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + throws SQLException; + + @Override + public ResultSet getColumnPrivileges( + String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + throws SQLException { + Statement stmt = this.connection.createStatement(); + + String sql = SHOW_DATABASES_SQL; + if (catalog != null && !catalog.isEmpty()) { + if (catalog.contains("%")) { + catalog = catalog.replace("%", "*"); + } + sql = sql + " " + catalog; + } else if (schemaPattern != null && !schemaPattern.isEmpty()) { + if (schemaPattern.contains("%")) { + schemaPattern = schemaPattern.replace("%", "*"); + } + sql = sql + " " + schemaPattern; + } + if (((catalog != null && !catalog.isEmpty()) + || schemaPattern != null && !schemaPattern.isEmpty()) + && tableNamePattern != null + && !tableNamePattern.isEmpty()) { + if (tableNamePattern.contains("%")) { + tableNamePattern = tableNamePattern.replace("%", "*"); + } + sql = sql + "." + tableNamePattern; + } + + if (((catalog != null && !catalog.isEmpty()) + || schemaPattern != null && !schemaPattern.isEmpty()) + && tableNamePattern != null + && !tableNamePattern.isEmpty() + && columnNamePattern != null + && !columnNamePattern.isEmpty()) { + sql = sql + "." + columnNamePattern; + } + ResultSet rs; + try { + rs = stmt.executeQuery(sql); + } catch (SQLException e) { + stmt.close(); + throw e; + } + Field[] fields = new Field[8]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", GRANTOR, "TEXT"); + fields[5] = new Field("", GRANTEE, "TEXT"); + fields[6] = new Field("", PRIVILEGE, "TEXT"); + fields[7] = new Field("", IS_GRANTABLE, "TEXT"); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT); + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + while (rs.next()) { + List valuesInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + if (i < 4) { + valuesInRow.add(rs.getString(1)); + } else if (i == 5) { + valuesInRow.add(getUserName()); + } else if (i == 6) { + valuesInRow.add(""); + } else if (i == 7) { + valuesInRow.add("NO"); + } else { + valuesInRow.add(""); + } + } + valuesList.add(valuesInRow); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error("Convert tsBlock error when get column privileges: {}", e.getMessage()); + } finally { + close(rs, stmt); + } + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) + throws SQLException { + Statement stmt = this.connection.createStatement(); + + String sql = SHOW_DATABASES_SQL; + if (catalog != null && !catalog.isEmpty()) { + if (catalog.contains("%")) { + catalog = catalog.replace("%", "*"); + } + sql = sql + " " + catalog; + } else if (schemaPattern != null && !schemaPattern.isEmpty()) { + if (schemaPattern.contains("%")) { + schemaPattern = schemaPattern.replace("%", "*"); + } + sql = sql + " " + schemaPattern; + } + if (((catalog != null && !catalog.isEmpty()) + || schemaPattern != null && !schemaPattern.isEmpty()) + && tableNamePattern != null + && !tableNamePattern.isEmpty()) { + if (tableNamePattern.contains("%")) { + tableNamePattern = tableNamePattern.replace("%", "*"); + } + sql = sql + "." + tableNamePattern; + } + ResultSet rs; + try { + rs = stmt.executeQuery(sql); + } catch (SQLException e) { + stmt.close(); + throw e; + } + Field[] fields = new Field[8]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", GRANTOR, "TEXT"); + fields[5] = new Field("", GRANTEE, "TEXT"); + fields[6] = new Field("", PRIVILEGE, "TEXT"); + fields[7] = new Field("", IS_GRANTABLE, "TEXT"); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT); + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + while (rs.next()) { + List valueInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + if (i < 4) { + valueInRow.add(rs.getString(1)); + } else if (i == 5) { + valueInRow.add(getUserName()); + } else if (i == 6) { + valueInRow.add(""); + } else if (i == 7) { + valueInRow.add("NO"); + } else { + valueInRow.add(""); + } + } + valuesList.add(valueInRow); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(rs, stmt); + } + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getBestRowIdentifier( + String catalog, String schema, String table, int scope, boolean nullable) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[8]; + fields[0] = new Field("", SCOPE, INT32); + fields[1] = new Field("", COLUMN_NAME, "TEXT"); + fields[2] = new Field("", DATA_TYPE, INT32); + fields[3] = new Field("", TYPE_NAME, "TEXT"); + fields[4] = new Field("", COLUMN_SIZE, INT32); + fields[5] = new Field("", BUFFER_LENGTH, INT32); + fields[6] = new Field("", DECIMAL_DIGITS, INT32); + fields[7] = new Field("", PSEUDO_COLUMN, INT32); + + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get best row identifier error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getVersionColumns(String catalog, String schema, String table) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[8]; + fields[0] = new Field("", SCOPE, INT32); + fields[1] = new Field("", COLUMN_NAME, "TEXT"); + fields[2] = new Field("", DATA_TYPE, INT32); + fields[3] = new Field("", TYPE_NAME, "TEXT"); + fields[4] = new Field("", COLUMN_SIZE, INT32); + fields[5] = new Field("", BUFFER_LENGTH, INT32); + fields[6] = new Field("", DECIMAL_DIGITS, INT32); + fields[7] = new Field("", PSEUDO_COLUMN, INT32); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get version columns error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + public abstract ResultSet getPrimaryKeys(String catalog, String schema, String table) + throws SQLException; + + @Override + public ResultSet getImportedKeys(String catalog, String schema, String table) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[14]; + fields[0] = new Field("", PKTABLE_CAT, "TEXT"); + fields[1] = new Field("", PKTABLE_SCHEM, INT32); + fields[2] = new Field("", PKTABLE_NAME, "TEXT"); + fields[3] = new Field("", PKCOLUMN_NAME, "TEXT"); + fields[4] = new Field("", FKTABLE_CAT, "TEXT"); + fields[5] = new Field("", FKTABLE_SCHEM, "TEXT"); + fields[6] = new Field("", FKTABLE_NAME, "TEXT"); + fields[7] = new Field("", FKCOLUMN_NAME, "TEXT"); + fields[8] = new Field("", KEY_SEQ, INT32); + fields[9] = new Field("", UPDATE_RULE, INT32); + fields[10] = new Field("", DELETE_RULE, INT32); + fields[11] = new Field("", FK_NAME, "TEXT"); + fields[12] = new Field("", PK_NAME, "TEXT"); + fields[13] = new Field("", DEFERRABILITY, INT32); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get import keys error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getExportedKeys(String catalog, String schema, String table) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[14]; + fields[0] = new Field("", PKTABLE_CAT, "TEXT"); + fields[1] = new Field("", PKTABLE_SCHEM, INT32); + fields[2] = new Field("", PKTABLE_NAME, "TEXT"); + fields[3] = new Field("", PKCOLUMN_NAME, "TEXT"); + fields[4] = new Field("", FKTABLE_CAT, "TEXT"); + fields[5] = new Field("", FKTABLE_SCHEM, "TEXT"); + fields[6] = new Field("", FKTABLE_NAME, "TEXT"); + fields[7] = new Field("", FKCOLUMN_NAME, "TEXT"); + fields[8] = new Field("", KEY_SEQ, INT32); + fields[9] = new Field("", UPDATE_RULE, INT32); + fields[10] = new Field("", DELETE_RULE, INT32); + fields[11] = new Field("", FK_NAME, "TEXT"); + fields[12] = new Field("", PK_NAME, "TEXT"); + fields[13] = new Field("", DEFERRABILITY, INT32); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get exported keys error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getCrossReference( + String parentCatalog, + String parentSchema, + String parentTable, + String foreignCatalog, + String foreignSchema, + String foreignTable) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[14]; + fields[0] = new Field("", PKTABLE_CAT, "TEXT"); + fields[1] = new Field("", PKTABLE_SCHEM, "TEXT"); + fields[2] = new Field("", PKTABLE_NAME, "TEXT"); + fields[3] = new Field("", PKCOLUMN_NAME, "TEXT"); + fields[4] = new Field("", FKTABLE_CAT, "TEXT"); + fields[5] = new Field("", FKTABLE_SCHEM, "TEXT"); + fields[6] = new Field("", FKTABLE_NAME, "TEXT"); + fields[7] = new Field("", FKCOLUMN_NAME, "TEXT"); + fields[8] = new Field("", KEY_SEQ, "TEXT"); + fields[9] = new Field("", UPDATE_RULE, "TEXT"); + fields[10] = new Field("", DELETE_RULE, "TEXT"); + fields[11] = new Field("", FK_NAME, "TEXT"); + fields[12] = new Field("", PK_NAME, "TEXT"); + fields[13] = new Field("", DEFERRABILITY, "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get cross reference error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getTypeInfo() throws SQLException { + Statement stmt = connection.createStatement(); + Field[] fields = new Field[18]; + fields[0] = new Field("", TYPE_NAME, "TEXT"); + fields[1] = new Field("", DATA_TYPE, INT32); + fields[2] = new Field("", PRECISION, INT32); + fields[3] = new Field("", "LITERAL_PREFIX", "TEXT"); + fields[4] = new Field("", "LITERAL_SUFFIX", "TEXT"); + fields[5] = new Field("", "CREATE_PARAMS", "TEXT"); + fields[6] = new Field("", NULLABLE, INT32); + fields[7] = new Field("", "CASE_SENSITIVE", BOOLEAN); + fields[8] = new Field("", "SEARCHABLE", "TEXT"); + fields[9] = new Field("", "UNSIGNED_ATTRIBUTE", BOOLEAN); + fields[10] = new Field("", "FIXED_PREC_SCALE", BOOLEAN); + fields[11] = new Field("", "AUTO_INCREMENT", BOOLEAN); + fields[12] = new Field("", "LOCAL_TYPE_NAME", "TEXT"); + fields[13] = new Field("", "MINIMUM_SCALE", INT32); + fields[14] = new Field("", "MAXIMUM_SCALE", INT32); + fields[15] = new Field("", SQL_DATA_TYPE, INT32); + fields[16] = new Field("", SQL_DATETIME_SUB, INT32); + fields[17] = new Field("", NUM_PREC_RADIX, INT32); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.BOOLEAN, + TSDataType.TEXT, + TSDataType.BOOLEAN, + TSDataType.BOOLEAN, + TSDataType.BOOLEAN, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32); + List listValSub1 = + Arrays.asList( + INT32, + Types.INTEGER, + 10, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub2 = + Arrays.asList( + INT64, + Types.BIGINT, + 19, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub3 = + Arrays.asList( + BOOLEAN, + Types.BOOLEAN, + 1, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub4 = + Arrays.asList( + FLOAT, + Types.FLOAT, + 38, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub5 = + Arrays.asList( + DOUBLE, + Types.DOUBLE, + 308, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub6 = + Arrays.asList( + "TEXT", + Types.LONGVARCHAR, + 64, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub7 = + Arrays.asList( + STRING, + Types.VARCHAR, + 64, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub8 = + Arrays.asList( + BLOB, Types.BLOB, 64, "", "", "", 1, true, "", false, true, false, "", 0, 10, 0, 0, 10); + List listValSub9 = + Arrays.asList( + TIMESTAMP, + Types.TIMESTAMP, + 3, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List listValSub10 = + Arrays.asList( + DATE, Types.DATE, 0, "", "", "", 1, true, "", false, true, false, "", 0, 10, 0, 0, 10); + List> valuesList = + Arrays.asList( + listValSub1, + listValSub2, + listValSub3, + listValSub4, + listValSub5, + listValSub6, + listValSub7, + listValSub8, + listValSub9, + listValSub10); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getIndexInfo( + String catalog, String schema, String table, boolean unique, boolean approximate) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[14]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", "NON_UNIQUE", "TEXT"); + fields[4] = new Field("", "INDEX_QUALIFIER", "TEXT"); + fields[5] = new Field("", "INDEX_NAME", "TEXT"); + fields[6] = new Field("", "TYPE", "TEXT"); + fields[7] = new Field("", ORDINAL_POSITION, "TEXT"); + fields[8] = new Field("", COLUMN_NAME, "TEXT"); + fields[9] = new Field("", "ASC_OR_DESC", "TEXT"); + fields[10] = new Field("", "CARDINALITY", "TEXT"); + fields[11] = new Field("", "PAGES", "TEXT"); + fields[12] = new Field("", PK_NAME, "TEXT"); + fields[13] = new Field("", "FILTER_CONDITION", "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get index info error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public boolean supportsResultSetType(int type) throws SQLException { + return ResultSet.FETCH_FORWARD == type || ResultSet.TYPE_FORWARD_ONLY == type; + } + + @Override + public boolean supportsResultSetConcurrency(int type, int concurrency) throws SQLException { + return false; + } + + @Override + public boolean ownUpdatesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean ownDeletesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean ownInsertsAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean othersUpdatesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean othersDeletesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean othersInsertsAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean updatesAreDetected(int type) throws SQLException { + return false; + } + + @Override + public boolean deletesAreDetected(int type) throws SQLException { + return false; + } + + @Override + public boolean insertsAreDetected(int type) throws SQLException { + return false; + } + + @Override + public boolean supportsBatchUpdates() throws SQLException { + return true; + } + + @Override + public ResultSet getUDTs( + String catalog, String schemaPattern, String typeNamePattern, int[] types) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[7]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", "CLASS_NAME", "TEXT"); + fields[4] = new Field("", DATA_TYPE, INT32); + fields[5] = new Field("", REMARKS, "TEXT"); + fields[6] = new Field("", "BASE_TYPE", "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get UDTS error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public Connection getConnection() throws SQLException { + return connection; + } + + @Override + public boolean supportsSavepoints() throws SQLException { + return false; + } + + @Override + public boolean supportsNamedParameters() throws SQLException { + return false; + } + + @Override + public boolean supportsMultipleOpenResults() throws SQLException { + return false; + } + + @Override + public boolean supportsGetGeneratedKeys() throws SQLException { + return false; + } + + @Override + public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[6]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", "SUPERTYPE_CAT", "TEXT"); + fields[4] = new Field("", "SUPERTYPE_SCHEM", "TEXT"); + fields[5] = new Field("", "SUPERTYPE_NAME", "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get super types error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[4]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", "SUPERTABLE_NAME", "TEXT"); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get super tables error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getAttributes( + String catalog, String schemaPattern, String typeNamePattern, String attributeNamePattern) + throws SQLException { + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + Statement stmt = connection.createStatement(); + try { + Field[] fields = new Field[21]; + fields[0] = new Field("", TYPE_CAT, "TEXT"); + fields[1] = new Field("", TYPE_SCHEM, "TEXT"); + fields[2] = new Field("", TYPE_NAME, "TEXT"); + fields[3] = new Field("", "ATTR_NAME", "TEXT"); + fields[4] = new Field("", DATA_TYPE, INT32); + fields[5] = new Field("", "ATTR_TYPE_NAME", "TEXT"); + fields[6] = new Field("", "ATTR_SIZE", INT32); + fields[7] = new Field("", DECIMAL_DIGITS, INT32); + fields[8] = new Field("", NUM_PREC_RADIX, INT32); + fields[9] = new Field("", NULLABLE, INT32); + fields[10] = new Field("", REMARKS, "TEXT"); + fields[11] = new Field("", "ATTR_DEF", "TEXT"); + fields[12] = new Field("", SQL_DATA_TYPE, INT32); + fields[13] = new Field("", SQL_DATETIME_SUB, INT32); + fields[14] = new Field("", CHAR_OCTET_LENGTH, INT32); + fields[15] = new Field("", ORDINAL_POSITION, INT32); + fields[16] = new Field("", IS_NULLABLE, "TEXT"); + fields[17] = new Field("", "SCOPE_CATALOG", "TEXT"); + fields[18] = new Field("", "SCOPE_SCHEMA", "TEXT"); + fields[19] = new Field("", "SCOPE_TABLE", "TEXT"); + fields[20] = new Field("", "SOURCE_DATA_TYPE", INT32); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + } catch (Exception e) { + LOGGER.error("Get attributes error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + false, + client, + null, + -1, + sessionId, + null, + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public boolean supportsResultSetHoldability(int holdability) throws SQLException { + return ResultSet.HOLD_CURSORS_OVER_COMMIT == holdability; + } + + @Override + public int getResultSetHoldability() throws SQLException { + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public int getDatabaseMajorVersion() throws SQLException { + int majorVersion = 0; + try { + String version = client.getProperties().getVersion(); + String[] versions = version.split("\\."); + if (versions.length >= 2) { + majorVersion = Integer.parseInt(versions[0]); + } + } catch (TException e) { + LOGGER.error("Get database major version error: {}", e.getMessage()); + } + return majorVersion; + } + + @Override + public int getDatabaseMinorVersion() throws SQLException { + int minorVersion = 0; + try { + String version = client.getProperties().getVersion(); + String[] versions = version.split("\\."); + if (versions.length >= 2) { + minorVersion = Integer.parseInt(versions[1]); + } + } catch (TException e) { + LOGGER.error("Get database minor version error: {}", e.getMessage()); + } + return minorVersion; + } + + @Override + public int getJDBCMajorVersion() throws SQLException { + return 4; + } + + @Override + public int getJDBCMinorVersion() throws SQLException { + return 3; + } + + @Override + public int getSQLStateType() throws SQLException { + return 0; + } + + @Override + public boolean locatorsUpdateCopy() throws SQLException { + return false; + } + + @Override + public boolean supportsStatementPooling() throws SQLException { + return false; + } + + @Override + public RowIdLifetime getRowIdLifetime() throws SQLException { + return RowIdLifetime.ROWID_UNSUPPORTED; + } + + @Override + public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { + Statement stmt = this.connection.createStatement(); + ResultSet rs; + try { + rs = stmt.executeQuery(SHOW_DATABASES_SQL); + } catch (SQLException e) { + stmt.close(); + throw e; + } + Field[] fields = new Field[2]; + fields[0] = new Field("", TABLE_SCHEM, "TEXT"); + fields[1] = new Field("", "TABLE_CATALOG", "TEXT"); + + List tsDataTypeList = Arrays.asList(TSDataType.TEXT, TSDataType.TEXT); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + while (rs.next()) { + String database = rs.getString(0); + List valueInRow = new ArrayList<>(); + valueInRow.add(database); + valueInRow.add(""); + valuesList.add(valueInRow); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(rs, stmt); + } + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { + return false; + } + + @Override + public boolean autoCommitFailureClosesAllResultSets() throws SQLException { + return false; + } + + @Override + public ResultSet getClientInfoProperties() throws SQLException { + Statement stmt = this.connection.createStatement(); + ResultSet rs; + try { + rs = stmt.executeQuery(SHOW_DATABASES_SQL); + } catch (SQLException e) { + stmt.close(); + throw e; + } + + Field[] fields = new Field[4]; + fields[0] = new Field("", "NAME", "TEXT"); + fields[1] = new Field("", "MAX_LEN", INT32); + fields[2] = new Field("", "DEFAULT_VALUE", INT32); + fields[3] = new Field("", "DESCRIPTION", "TEXT"); + List tsDataTypeList = + Arrays.asList(TSDataType.TEXT, TSDataType.INT32, TSDataType.INT32, TSDataType.TEXT); + List values = Arrays.asList("fetch_size", 10, 10, ""); + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + valuesList.add(values); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error("Convert tsBlock error when get client info properties: {}", e.getMessage()); + } finally { + close(rs, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) + throws SQLException { + Statement stmt = connection.createStatement(); + ResultSet rs; + try { + rs = stmt.executeQuery(SHOW_FUNCTIONS); + } catch (SQLException e) { + stmt.close(); + throw e; + } + + Field[] fields = new Field[6]; + fields[0] = new Field("", FUNCTION_CAT, "TEXT"); + fields[1] = new Field("", FUNCTION_SCHEM, "TEXT"); + fields[2] = new Field("", FUNCTION_NAME, "TEXT"); + fields[3] = new Field("", REMARKS, "TEXT"); + fields[4] = new Field("", FUNCTION_TYPE, INT32); + fields[5] = new Field("", SPECIFIC_NAME, "TEXT"); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT); + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + while (rs.next()) { + List valueInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + if (i == 2) { + valueInRow.add(rs.getString(1)); + } else if (i == 4) { + valueInRow.add(0); + } else { + valueInRow.add(""); + } + } + valuesList.add(valueInRow); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error("Convert tsBlock error when get functions: {}", e.getMessage()); + } finally { + close(rs, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getFunctionColumns( + String catalog, String schemaPattern, String functionNamePattern, String columnNamePattern) + throws SQLException { + Statement stmt = connection.createStatement(); + ResultSet rs; + try { + rs = stmt.executeQuery(SHOW_FUNCTIONS); + } catch (SQLException e) { + stmt.close(); + throw e; + } + Field[] fields = new Field[17]; + fields[0] = new Field("", FUNCTION_CAT, "TEXT"); + fields[1] = new Field("", FUNCTION_SCHEM, "TEXT"); + fields[2] = new Field("", FUNCTION_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", COLUMN_TYPE, INT32); + fields[5] = new Field("", DATA_TYPE, INT32); + fields[6] = new Field("", TYPE_NAME, "TEXT"); + fields[7] = new Field("", PRECISION, INT32); + fields[8] = new Field("", LENGTH, INT32); + fields[9] = new Field("", SCALE, INT32); + fields[10] = new Field("", RADIX, INT32); + fields[11] = new Field("", NULLABLE, INT32); + fields[12] = new Field("", REMARKS, "TEXT"); + fields[13] = new Field("", CHAR_OCTET_LENGTH, INT32); + fields[14] = new Field("", ORDINAL_POSITION, INT32); + fields[15] = new Field("", IS_NULLABLE, "TEXT"); + fields[16] = new Field("", SPECIFIC_NAME, "TEXT"); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT); + + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + while (rs.next()) { + List valuesInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + if (i == 2) { + valuesInRow.add(rs.getString(1)); + } else if (INT32.equals(fields[i].getSqlType())) { + valuesInRow.add(0); + } else { + valuesInRow.add(""); + } + } + valuesList.add(valuesInRow); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error("Convert tsBlock error when get function columns: {}", e.getMessage()); + } finally { + close(rs, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getPseudoColumns( + String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + throws SQLException { + Statement stmt = connection.createStatement(); + Field[] fields = new Field[12]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", DATA_TYPE, INT32); + fields[5] = new Field("", COLUMN_SIZE, INT32); + fields[6] = new Field("", DECIMAL_DIGITS, INT32); + fields[7] = new Field("", NUM_PREC_RADIX, INT32); + fields[8] = new Field("", "COLUMN_USAGE", "TEXT"); + fields[9] = new Field("", REMARKS, "TEXT"); + fields[10] = new Field("", CHAR_OCTET_LENGTH, INT32); + fields[11] = new Field("", IS_NULLABLE, "TEXT"); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT); + + List value = + Arrays.asList( + catalog, catalog, tableNamePattern, "times", Types.BIGINT, 1, 0, 2, "", "", 13, "NO"); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(Collections.singletonList(value), tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + 0, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public boolean generatedKeyAlwaysReturned() throws SQLException { + return true; + } + + @Override + public T unwrap(Class arg0) throws SQLException { + throw new SQLException(METHOD_NOT_SUPPORTED_STRING); + } + + @Override + public boolean isWrapperFor(Class arg0) throws SQLException { + throw new SQLException(METHOD_NOT_SUPPORTED_STRING); + } +} diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java index 06e9221515530..857c41603acf5 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java @@ -20,6 +20,7 @@ package org.apache.iotdb.jdbc; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.jdbc.relational.IoTDBRelationalDatabaseMetadata; import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.StatementExecutionException; @@ -31,6 +32,7 @@ import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; +import org.apache.commons.lang3.StringUtils; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TCompactProtocol; @@ -70,6 +72,7 @@ public class IoTDBConnection implements Connection { TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3; private static final String NOT_SUPPORT_PREPARE_CALL = "Does not support prepareCall"; private static final String NOT_SUPPORT_PREPARE_STATEMENT = "Does not support prepareStatement"; + private static final String APACHE_IOTDB = "Apache IoTDB"; private IClientRPCService.Iface client = null; private long sessionId = -1; private IoTDBConnectionParams params; @@ -101,6 +104,18 @@ public String getUserName() { private String userName; + // default is tree + public String getSqlDialect() { + if (params != null && StringUtils.isNotBlank(params.getSqlDialect())) { + return params.getSqlDialect(); + } else { + return "tree"; + } + } + + // ms is 1_000, us is 1_000_000, ns is 1_000_000_000 + private int timeFactor = 1_000; + public IoTDBConnection() { // allowed to create an instance without parameter input. } @@ -132,6 +147,10 @@ public String getUrl() { return url; } + public IoTDBConnectionParams getParams() { + return params; + } + @Override public boolean isWrapperFor(Class arg0) throws SQLException { throw new SQLException("Does not support isWrapperFor"); @@ -244,12 +263,32 @@ public void setAutoCommit(boolean arg0) { @Override public String getCatalog() { - return "Apache IoTDB"; + return APACHE_IOTDB; } @Override public void setCatalog(String arg0) throws SQLException { - throw new SQLException("Does not support setCatalog"); + if (getSqlDialect().equals(Constant.TABLE_DIALECT)) { + if (APACHE_IOTDB.equals(arg0)) { + return; + } + for (String str : IoTDBRelationalDatabaseMetadata.allIotdbTableSQLKeywords) { + if (arg0.equalsIgnoreCase(str)) { + arg0 = "\"" + arg0 + "\""; + } + } + + Statement stmt = this.createStatement(); + String sql = "USE " + arg0; + boolean rs; + try { + rs = stmt.execute(sql); + } catch (SQLException e) { + stmt.close(); + logger.error("Use database error: {}", e.getMessage()); + throw e; + } + } } @Override @@ -282,6 +321,9 @@ public DatabaseMetaData getMetaData() throws SQLException { if (isClosed) { throw new SQLException("Cannot create statement because connection is closed"); } + if (getSqlDialect().equals(Constant.TABLE_DIALECT)) { + return new IoTDBRelationalDatabaseMetadata(this, getClient(), sessionId, zoneId); + } return new IoTDBDatabaseMetadata(this, getClient(), sessionId, zoneId); } @@ -292,12 +334,33 @@ public int getNetworkTimeout() { @Override public String getSchema() throws SQLException { + if (getSqlDialect().equals(Constant.TABLE_DIALECT)) { + return getDatabase(); + } throw new SQLException("Does not support getSchema"); } @Override public void setSchema(String arg0) throws SQLException { - throw new SQLException("Does not support setSchema"); + // changeDefaultDatabase(arg0); + if (getSqlDialect().equals(Constant.TABLE_DIALECT)) { + for (String str : IoTDBRelationalDatabaseMetadata.allIotdbTableSQLKeywords) { + if (arg0.equalsIgnoreCase(str)) { + arg0 = "\"" + arg0 + "\""; + } + } + + Statement stmt = this.createStatement(); + String sql = "USE " + arg0; + boolean rs; + try { + rs = stmt.execute(sql); + } catch (SQLException e) { + stmt.close(); + logger.error("Use database error: {}", e.getMessage()); + throw e; + } + } } @Override @@ -497,7 +560,9 @@ private void openSession() throws SQLException { openReq.setUsername(params.getUsername()); openReq.setPassword(params.getPassword()); openReq.setZoneId(getTimeZone()); - openReq.putToConfiguration("version", params.getVersion().toString()); + openReq.putToConfiguration(Config.VERSION, params.getVersion().toString()); + openReq.putToConfiguration(Config.SQL_DIALECT, params.getSqlDialect()); + params.getDb().ifPresent(db -> openReq.putToConfiguration(Config.DATABASE, db)); TSOpenSessionResp openResp = null; try { @@ -506,6 +571,7 @@ private void openSession() throws SQLException { // validate connection RpcUtils.verifySuccess(openResp.getStatus()); + this.timeFactor = RpcUtils.getTimeFactor(openResp); if (protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { logger.warn( "Protocol differ, Client version is {}}, but Server version is {}", @@ -542,7 +608,7 @@ private void openSession() throws SQLException { isClosed = false; } - boolean reconnect() { + public boolean reconnect() { boolean flag = false; for (int i = 1; i <= Config.RETRY_NUM; i++) { try { @@ -592,4 +658,23 @@ public void setTimeZone(String timeZone) throws TException, IoTDBSQLException { public ServerProperties getServerProperties() throws TException { return getClient().getProperties(); } + + protected void changeDefaultDatabase(String database) { + params.setDb(database); + } + + protected void mayChangeDefaultSqlDialect(String sqlDialect) { + if (!sqlDialect.equals(params.getSqlDialect())) { + params.setSqlDialect(sqlDialect); + params.setDb(null); + } + } + + public int getTimeFactor() { + return timeFactor; + } + + public String getDatabase() { + return params.getDb().orElse(null); + } } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java index 07700bad2b1c4..1112caabd4e20 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java @@ -25,6 +25,9 @@ import java.nio.charset.Charset; import java.time.ZoneId; +import java.util.Optional; + +import static org.apache.iotdb.jdbc.Constant.TREE; public class IoTDBConnectionParams { @@ -49,6 +52,10 @@ public class IoTDBConnectionParams { private String trustStore; private String trustStorePwd; + private String sqlDialect = TREE; + + private String db; + public IoTDBConnectionParams(String url) { this.jdbcUriString = url; } @@ -176,4 +183,20 @@ public String getTrustStorePwd() { public void setTrustStorePwd(String trustStorePwd) { this.trustStorePwd = trustStorePwd; } + + public String getSqlDialect() { + return sqlDialect; + } + + public void setSqlDialect(String sqlDialect) { + this.sqlDialect = sqlDialect; + } + + public Optional getDb() { + return Optional.ofNullable(db); + } + + public void setDb(String db) { + this.db = db; + } } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDataSourceFactory.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDataSourceFactory.java index 81259278dc8e9..8e601d5df6fa5 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDataSourceFactory.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDataSourceFactory.java @@ -45,16 +45,13 @@ public void setProperties(IoTDBDataSource ds, Properties prop) { String url = (String) properties.remove(DataSourceFactory.JDBC_URL); if (url != null) { ds.setUrl(url); - logger.info("URL set {}", url); } String user = (String) properties.remove(DataSourceFactory.JDBC_USER); ds.setUser(user); - logger.info("User set {}", user); String password = (String) properties.remove(DataSourceFactory.JDBC_PASSWORD); ds.setPassword(password); - logger.info("Password set {}", password); logger.info("Remaining properties {}", properties.size()); diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java index 4ed80e9a0e14b..a3963337c3bb2 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java @@ -27,24 +27,15 @@ import org.apache.commons.lang3.StringUtils; import org.apache.thrift.TException; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.block.TsBlock; -import org.apache.tsfile.read.common.block.TsBlockBuilder; -import org.apache.tsfile.read.common.block.column.TsBlockSerde; -import org.apache.tsfile.utils.Binary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; -import java.sql.Connection; -import java.sql.DatabaseMetaData; import java.sql.ResultSet; -import java.sql.RowIdLifetime; import java.sql.SQLException; import java.sql.Statement; -import java.sql.Types; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -55,894 +46,255 @@ import java.util.Map; import java.util.TreeMap; -public class IoTDBDatabaseMetadata implements DatabaseMetaData { +public class IoTDBDatabaseMetadata extends IoTDBAbstractDatabaseMetadata { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(IoTDBDatabaseMetadata.class); - private IoTDBConnection connection; - private IClientRPCService.Iface client; private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDatabaseMetadata.class); - private static final String METHOD_NOT_SUPPORTED_STRING = "Method not supported"; - // when running the program in IDE, we can not get the version info using - // getImplementationVersion() + private static final String DATABASE_VERSION = IoTDBDatabaseMetadata.class.getPackage().getImplementationVersion() != null ? IoTDBDatabaseMetadata.class.getPackage().getImplementationVersion() : "UNKNOWN"; - private long sessionId; - private ZoneId zoneId; - private static String sqlKeywordsThatArentSQL92; - private static TsBlockSerde serde = new TsBlockSerde(); - - private static final String SHOW_DATABASES_SQL = "SHOW DATABASES "; - - private static final String PRECISION = "PRECISION"; - - private static final String PRIMARY = "PRIMARY"; - - private static final String DOUBLE = "DOUBLE"; - private static final String BOOLEAN = "BOOLEAN"; - private static final String FLOAT = "FLOAT"; - private static final String INT32 = "INT32"; - - private static final String INT64 = "INT64"; - - private static final String TYPE_CAT = "TYPE_CAT"; - - private static final String TYPE_NAME = "TYPE_NAME"; - - private static final String REMARKS = "REMARKS"; - - private static final String IS_NULLABLE = "IS_NULLABLE"; - - private static final String COLUMN_NAME = "COLUMN_NAME"; - - private static final String TABLE_CAT = "TABLE_CAT"; - - private static final String TABLE_SCHEM = "TABLE_SCHEM"; - - private static final String TABLE_NAME = "TABLE_NAME"; - - private static final String PKTABLE_CAT = "PKTABLE_CAT"; - - private static final String PKTABLE_SCHEM = "PKTABLE_SCHEM"; - private static final String PKTABLE_NAME = "PKTABLE_NAME"; - - private static final String PKCOLUMN_NAME = "PKCOLUMN_NAME"; - - private static final String FKTABLE_CAT = "FKTABLE_CAT"; - - private static final String FKTABLE_SCHEM = "FKTABLE SCHEM"; - - private static final String FKTABLE_NAME = "FKTABLE_NAME"; - - private static final String FKCOLUMN_NAME = "FKCOLUMN_NAME"; - - private static final String KEY_SEQ = "KEY_SEQ"; - - private static final String DELETE_RULE = "DELETE_RULE"; - - private static final String FK_NAME = "FK_NAME"; - - private static final String PK_NAME = "PK_NAME"; - - private static final String DEFERRABILITY = "DEFERRABILITY"; - - private static final String SPECIFIC_NAME = "SPECIFIC_NAME"; - - private static final String TABLE_TYPE = "TABLE_TYPE"; - private static final String SHOW_FUNCTIONS = "show functions"; - private static final String DECIMAL_DIGITS = "DECIMAL_DIGITS"; - private static final String SQL_DATETIME_SUB = "SQL_DATETIME_SUB"; - private static final String CHAR_OCTET_LENGTH = "CHAR_OCTET_LENGTH"; - private static final String COLUMN_SIZE = "COLUMN_SIZE"; - private static final String BUFFER_LENGTH = "BUFFER_LENGTH"; - private static final String DATA_TYPE = "DATA_TYPE"; - private static final String NUM_PREC_RADIX = "NUM_PREC_RADIX"; - private static final String SQL_DATA_TYPE = "SQL_DATA_TYPE"; - private static final String ORDINAL_POSITION = "ORDINAL_POSITION"; - private static final String NULLABLE = "NULLABLE"; - - private static final String CONVERT_ERROR_MSG = "convert tsBlock error: {}"; - - IoTDBDatabaseMetadata( - IoTDBConnection connection, IClientRPCService.Iface client, long sessionId, ZoneId zoneId) { - this.connection = connection; - this.client = client; - this.sessionId = sessionId; - this.zoneId = zoneId; - } + private static final String[] allIotdbSQLKeywords = { + "ALTER", + "ADD", + "ALIAS", + "ALL", + "AVG", + "ALIGN", + "ATTRIBUTES", + "AS", + "ASC", + "BY", + BOOLEAN, + "BITMAP", + "CREATE", + "CONFIGURATION", + "COMPRESSOR", + "CHILD", + "COUNT", + "COMPRESSION", + "CLEAR", + "CACHE", + "CONTAIN", + "CONCAT", + "DELETE", + "DEVICE", + "DESCRIBE", + "DATATYPE", + DOUBLE, + "DIFF", + "DROP", + "DEVICES", + "DISABLE", + "DESC", + "ENCODING", + "FROM", + "FILL", + FLOAT, + "FLUSH", + "FIRST_VALUE", + "FULL", + "FALSE", + "FOR", + "FUNCTION", + "FUNCTIONS", + "GRANT", + "GROUP", + "GORILLA", + "GLOBAL", + "GZIP", + "INSERT", + "INTO", + INT32, + INT64, + "INDEX", + "INFO", + "KILL", + "LIMIT", + "LINEAR", + "LABEL", + "LINK", + "LIST", + "LOAD", + "LEVEL", + "LAST_VALUE", + "LAST", + "LZO", + "LZ4", + "ZSTD", + "LZMA2", + "LATEST", + "LIKE", + "MAX_BY", + "MIN_BY", + "METADATA", + "MOVE", + "MIN_TIME", + "MAX_TIME", + "MIN_VALUE", + "MAX_VALUE", + "NOW", + "NODES", + "ORDER", + "OFFSET", + "ON", + "OFF", + "OF", + "PROCESSLIST", + "PREVIOUS", + "PREVIOUSUNTILLAST", + "PROPERTY", + "PLAIN", + "PLAIN_DICTIONARY", + "PRIVILEGES", + "PASSWORD", + "PATHS", + "PAA", + "PLA", + "PARTITION", + "QUERY", + "ROOT", + "RLE", + "REGULAR", + "ROLE", + "REVOKE", + "REMOVE", + "RENAME", + "SELECT", + "SHOW", + "SET", + "SLIMIT", + "SOFFSET", + "STDDEV", + "STDDEV_POP", + "STDDEV_SAMP", + "STORAGE", + "SUM", + "SNAPPY", + "SNAPSHOT", + "SCHEMA", + "TO", + "TIMESERIES", + "TIMESTAMP", + "TEXT", + "TS_2DIFF", + "TRACING", + "TTL", + "TASK", + "TIME", + "TAGS", + "TRUE", + "TEMPORARY", + "TOP", + "TOLERANCE", + "UPDATE", + "UNLINK", + "UPSERT", + "USING", + "USER", + "UNSET", + "UNCOMPRESSED", + "VALUES", + "VARIANCE", + "VAR_POP", + "VAR_SAMP", + "VERSION", + "WHERE", + "WITH", + "WATERMARK_EMBEDDING" + }; + + private static final String[] allIoTDBMathFunctions = { + "ABS", "ACOS", "ASIN", "ATAN", "CEIL", "COS", "COSH", "DEGREES", "EXP", "FLOOR", "LN", "LOG10", + "PI", "RADIANS", "ROUND", "SIGN", "SIN", "SINH", "SQRT", "TAN", "TANH" + }; static { - String[] allIotdbSQLKeywords = { - "ALTER", - "ADD", - "ALIAS", - "ALL", - "AVG", - "ALIGN", - "ATTRIBUTES", - "AS", - "ASC", - "BY", - BOOLEAN, - "BITMAP", - "CREATE", - "CONFIGURATION", - "COMPRESSOR", - "CHILD", - "COUNT", - "COMPRESSION", - "CLEAR", - "CACHE", - "CONTAIN", - "CONCAT", - "DELETE", - "DEVICE", - "DESCRIBE", - "DATATYPE", - DOUBLE, - "DIFF", - "DROP", - "DEVICES", - "DISABLE", - "DESC", - "ENCODING", - "FROM", - "FILL", - FLOAT, - "FLUSH", - "FIRST_VALUE", - "FULL", - "FALSE", - "FOR", - "FUNCTION", - "FUNCTIONS", - "GRANT", - "GROUP", - "GORILLA", - "GLOBAL", - "GZIP", - "INSERT", - "INTO", - INT32, - INT64, - "INDEX", - "INFO", - "KILL", - "LIMIT", - "LINEAR", - "LABEL", - "LINK", - "LIST", - "LOAD", - "LEVEL", - "LAST_VALUE", - "LAST", - "LZO", - "LZ4", - "ZSTD", - "LZMA2", - "LATEST", - "LIKE", - "MAX_BY", - "MIN_BY", - "METADATA", - "MOVE", - "MIN_TIME", - "MAX_TIME", - "MIN_VALUE", - "MAX_VALUE", - "NOW", - "NODES", - "ORDER", - "OFFSET", - "ON", - "OFF", - "OF", - "PROCESSLIST", - "PREVIOUS", - "PREVIOUSUNTILLAST", - "PROPERTY", - "PLAIN", - "PLAIN_DICTIONARY", - "PRIVILEGES", - "PASSWORD", - "PATHS", - "PAA", - "PLA", - "PARTITION", - "QUERY", - "ROOT", - "RLE", - "REGULAR", - "ROLE", - "REVOKE", - "REMOVE", - "RENAME", - "SELECT", - "SHOW", - "SET", - "SLIMIT", - "SOFFSET", - "STDDEV", - "STDDEV_POP", - "STDDEV_SAMP", - "STORAGE", - "SUM", - "SNAPPY", - "SNAPSHOT", - "SCHEMA", - "TO", - "TIMESERIES", - "TIMESTAMP", - "TEXT", - "TS_2DIFF", - "TRACING", - "TTL", - "TASK", - "TIME", - "TAGS", - "TRUE", - "TEMPORARY", - "TOP", - "TOLERANCE", - "UPDATE", - "UNLINK", - "UPSERT", - "USING", - "USER", - "UNSET", - "UNCOMPRESSED", - "VALUES", - "VARIANCE", - "VAR_POP", - "VAR_SAMP", - "VERSION", - "WHERE", - "WITH", - "WATERMARK_EMBEDDING" - }; - String[] sql92Keywords = { - "ABSOLUTE", - "EXEC", - "OVERLAPS", - "ACTION", - "EXECUTE", - "PAD", - "ADA", - "EXISTS", - "PARTIAL", - "ADD", - "EXTERNAL", - "PASCAL", - "ALL", - "EXTRACT", - "POSITION", - "ALLOCATE", - "FALSE", - PRECISION, - "ALTER", - "FETCH", - "PREPARE", - "AND", - "FIRST", - "PRESERVE", - "ANY", - FLOAT, - PRIMARY, - "ARE", - "FOR", - "PRIOR", - "AS", - "FOREIGN", - "PRIVILEGES", - "ASC", - "FORTRAN", - "PROCEDURE", - "ASSERTION", - "FOUND", - "PUBLIC", - "AT", - "FROM", - "READ", - "AUTHORIZATION", - "FULL", - "REAL", - "AVG", - "GET", - "REFERENCES", - "BEGIN", - "GLOBAL", - "RELATIVE", - "BETWEEN", - "GO", - "RESTRICT", - "BIT", - "GOTO", - "REVOKE", - "BIT_LENGTH", - "GRANT", - "RIGHT", - "BOTH", - "GROUP", - "ROLLBACK", - "BY", - "HAVING", - "ROWS", - "CASCADE", - "HOUR", - "SCHEMA", - "CASCADED", - "IDENTITY", - "SCROLL", - "CASE", - "IMMEDIATE", - "SECOND", - "CAST", - "IN", - "SECTION", - "CATALOG", - "INCLUDE", - "SELECT", - "CHAR", - "INDEX", - "SESSION", - "CHAR_LENGTH", - "INDICATOR", - "SESSION_USER", - "CHARACTER", - "INITIALLY", - "SET", - "CHARACTER_LENGTH", - "INNER", - "SIZE", - "CHECK", - "INPUT", - "SMALLINT", - "CLOSE", - "INSENSITIVE", - "SOME", - "COALESCE", - "INSERT", - "SPACE", - "COLLATE", - "INT", - "SQL", - "COLLATION", - "INTEGER", - "SQLCA", - "COLUMN", - "INTERSECT", - "SQLCODE", - "COMMIT", - "INTERVAL", - "SQLERROR", - "CONNECT", - "INTO", - "SQLSTATE", - "CONNECTION", - "IS", - "SQLWARNING", - "CONSTRAINT", - "ISOLATION", - "SUBSTRING", - "CONSTRAINTS", - "JOIN", - "SUM", - "CONTINUE", - "KEY", - "SYSTEM_USER", - "CONVERT", - "LANGUAGE", - "TABLE", - "CORRESPONDING", - "LAST", - "TEMPORARY", - "COUNT", - "LEADING", - "THEN", - "CREATE", - "LEFT", - "TIME", - "CROSS", - "LEVEL", - "TIMESTAMP", - "CURRENT", - "LIKE", - "TIMEZONE_HOUR", - "CURRENT_DATE", - "LOCAL", - "TIMEZONE_MINUTE", - "CURRENT_TIME", - "LOWER", - "TO", - "CURRENT_TIMESTAMP", - "MATCH", - "TRAILING", - "CURRENT_USER", - "MAX", - "TRANSACTION", - "CURSOR", - "MIN", - "TRANSLATE", - "DATE", - "MINUTE", - "TRANSLATION", - "DAY", - "MODULE", - "TRIM", - "DEALLOCATE", - "MONTH", - "TRUE", - "DEC", - "NAMES", - "UNION", - "DECIMAL", - "NATIONAL", - "UNIQUE", - "DECLARE", - "NATURAL", - "UNKNOWN", - "DEFAULT", - "NCHAR", - "UPDATE", - "DEFERRABLE", - "NEXT", - "UPPER", - "DEFERRED", - "NO", - "USAGE", - "DELETE", - "NONE", - "USER", - "DESC", - "NOT", - "USING", - "DESCRIBE", - "NULL", - "VALUE", - "DESCRIPTOR", - "NULLIF", - "VALUES", - "DIAGNOSTICS", - "NUMERIC", - "VARCHAR", - "DISCONNECT", - "OCTET_LENGTH", - "VARYING", - "DISTINCT", - "OF", - "VIEW", - "DOMAIN", - "ON", - "WHEN", - DOUBLE, - "ONLY", - "WHENEVER", - "DROP", - "OPEN", - "WHERE", - "ELSE", - "OPTION", - "WITH", - "END", - "OR", - "WORK", - "END-EXEC", - "ORDER", - "WRITE", - "ESCAPE", - "OUTER", - "YEAR", - "EXCEPT", - "OUTPUT", - "ZONE", - "EXCEPTION" - }; - TreeMap myKeywordMap = new TreeMap(); - for (int i = 0; i < allIotdbSQLKeywords.length; i++) { - myKeywordMap.put(allIotdbSQLKeywords[i], null); - } - HashMap sql92KeywordMap = new HashMap(sql92Keywords.length); - for (int j = 0; j < sql92Keywords.length; j++) { - sql92KeywordMap.put(sql92Keywords[j], null); - } - Iterator it = sql92KeywordMap.keySet().iterator(); - while (it.hasNext()) { - myKeywordMap.remove(it.next()); - } - StringBuilder keywordBuf = new StringBuilder(); - it = myKeywordMap.keySet().iterator(); - if (it.hasNext()) { - keywordBuf.append(it.next().toString()); - } - while (it.hasNext()) { - keywordBuf.append(","); - keywordBuf.append(it.next().toString()); - } - sqlKeywordsThatArentSQL92 = keywordBuf.toString(); - } - - @Override - public boolean isWrapperFor(Class arg0) throws SQLException { - throw new SQLException(METHOD_NOT_SUPPORTED_STRING); - } - - @Override - public T unwrap(Class iface) throws SQLException { - throw new SQLException(METHOD_NOT_SUPPORTED_STRING); - } - - @Override - public boolean allProceduresAreCallable() { - return false; - } - - @Override - public boolean allTablesAreSelectable() { - return true; - } - - @Override - public boolean autoCommitFailureClosesAllResultSets() { - return false; - } - - @Override - public boolean dataDefinitionCausesTransactionCommit() { - return false; - } - - @Override - public boolean dataDefinitionIgnoredInTransactions() { - return false; - } - - @Override - public boolean deletesAreDetected(int arg0) { - return true; - } - - @Override - public boolean doesMaxRowSizeIncludeBlobs() { - return false; // The return value is tentatively FALSE and may be adjusted later - } - - @Override - public boolean generatedKeyAlwaysReturned() { - return true; - } - - @Override - public long getMaxLogicalLobSize() { - return Long.MAX_VALUE; - } - - @Override - public boolean supportsRefCursors() { - return false; - } - - @Override - public ResultSet getAttributes(String arg0, String arg1, String arg2, String arg3) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); try { - Field[] fields = new Field[21]; - fields[0] = new Field("", TYPE_CAT, "TEXT"); - fields[1] = new Field("", "TYPE_SCHEM", "TEXT"); - fields[2] = new Field("", TYPE_NAME, "TEXT"); - fields[3] = new Field("", "ATTR_NAME", "TEXT"); - fields[4] = new Field("", DATA_TYPE, INT32); - fields[5] = new Field("", "ATTR_TYPE_NAME", "TEXT"); - fields[6] = new Field("", "ATTR_SIZE", INT32); - fields[7] = new Field("", DECIMAL_DIGITS, INT32); - fields[8] = new Field("", NUM_PREC_RADIX, INT32); - fields[9] = new Field("", "NULLABLE ", INT32); - fields[10] = new Field("", REMARKS, "TEXT"); - fields[11] = new Field("", "ATTR_DEF", "TEXT"); - fields[12] = new Field("", SQL_DATA_TYPE, INT32); - fields[13] = new Field("", SQL_DATETIME_SUB, INT32); - fields[14] = new Field("", CHAR_OCTET_LENGTH, INT32); - fields[15] = new Field("", ORDINAL_POSITION, INT32); - fields[16] = new Field("", IS_NULLABLE, "TEXT"); - fields[17] = new Field("", "SCOPE_CATALOG", "TEXT"); - fields[18] = new Field("", "SCOPE_SCHEMA", "TEXT"); - fields[19] = new Field("", "SCOPE_TABLE", "TEXT"); - fields[20] = new Field("", "SOURCE_DATA_TYPE", INT32); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); + TreeMap myKeywordMap = new TreeMap<>(); + for (String allIotdbSQLKeyword : allIotdbSQLKeywords) { + myKeywordMap.put(allIotdbSQLKeyword, null); } - } catch (Exception e) { - LOGGER.error("get attributes error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - @Override - public ResultSet getBestRowIdentifier( - String arg0, String arg1, String arg2, int arg3, boolean arg4) throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[8]; - fields[0] = new Field("", "SCOPE", INT32); - fields[1] = new Field("", COLUMN_NAME, "TEXT"); - fields[2] = new Field("", DATA_TYPE, INT32); - fields[3] = new Field("", TYPE_NAME, "TEXT"); - fields[4] = new Field("", COLUMN_SIZE, INT32); - fields[5] = new Field("", BUFFER_LENGTH, INT32); - fields[6] = new Field("", DECIMAL_DIGITS, INT32); - fields[7] = new Field("", "PSEUDO_COLUMN", INT32); + HashMap sql92KeywordMap = new HashMap<>(sql92Keywords.length); + for (String sql92Keyword : sql92Keywords) { + sql92KeywordMap.put(sql92Keyword, null); + } - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); + Iterator it = sql92KeywordMap.keySet().iterator(); + while (it.hasNext()) { + myKeywordMap.remove(it.next()); + } + + StringBuilder keywordBuf = new StringBuilder(); + it = myKeywordMap.keySet().iterator(); + if (it.hasNext()) { + keywordBuf.append(it.next()); + } + while (it.hasNext()) { + keywordBuf.append(","); + keywordBuf.append(it.next()); } + sqlKeywordsThatArentSQL92 = keywordBuf.toString(); + } catch (Exception e) { - LOGGER.error("get best row identifier error: {}", e.getMessage()); - } finally { - close(null, stmt); + LOGGER.error("Error when initializing SQL keywords: ", e); + throw new RuntimeException(e); } - - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); } - @Override - public String getCatalogSeparator() { - return "."; + public IoTDBDatabaseMetadata( + IoTDBConnection connection, IClientRPCService.Iface client, long sessionId, ZoneId zoneId) { + super(connection, client, sessionId, zoneId); } @Override - public String getCatalogTerm() { - return "database"; + public String getDriverVersion() throws SQLException { + return DATABASE_VERSION; } - @SuppressWarnings( - "squid:S2095") // ignore Use try-with-resources or close this "Statement" in a "finally" - // clause @Override - public ResultSet getCatalogs() throws SQLException { - Statement stmt = this.connection.createStatement(); - ResultSet rs; - try { - rs = stmt.executeQuery(SHOW_DATABASES_SQL); - } catch (SQLException e) { - stmt.close(); - throw e; - } - - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List tsDataTypeList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); - tsDataTypeList.add(TSDataType.TEXT); - - while (rs.next()) { - List values = new ArrayList<>(); - values.add(rs.getString(1)); - valuesList.add(values); - } - columnNameList.add(TYPE_CAT); - columnTypeList.add("TEXT"); - columnNameIndex.put(TYPE_CAT, 0); - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error("get cateLogs error: {}", e.getMessage()); - } finally { - close(rs, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); + public String getNumericFunctions() throws SQLException { + return String.join(",", allIoTDBMathFunctions); } - public static ByteBuffer convertTsBlock( - List> valuesList, List tsDataTypeList) throws IOException { - TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(tsDataTypeList); - for (List valuesInRow : valuesList) { - tsBlockBuilder.getTimeColumnBuilder().writeLong(0); - for (int j = 0; j < tsDataTypeList.size(); j++) { - TSDataType columnType = tsDataTypeList.get(j); - switch (columnType) { - case TEXT: - case STRING: - case BLOB: - tsBlockBuilder - .getColumnBuilder(j) - .writeBinary( - new Binary(valuesInRow.get(j).toString(), TSFileConfig.STRING_CHARSET)); - break; - case FLOAT: - tsBlockBuilder.getColumnBuilder(j).writeFloat((float) valuesInRow.get(j)); - break; - case INT32: - case DATE: - tsBlockBuilder.getColumnBuilder(j).writeInt((int) valuesInRow.get(j)); - break; - case INT64: - case TIMESTAMP: - tsBlockBuilder.getColumnBuilder(j).writeLong((long) valuesInRow.get(j)); - break; - case DOUBLE: - tsBlockBuilder.getColumnBuilder(j).writeDouble((double) valuesInRow.get(j)); - break; - case BOOLEAN: - tsBlockBuilder.getColumnBuilder(j).writeBoolean((boolean) valuesInRow.get(j)); - break; - default: - LOGGER.error("No data type was matched {}", columnType); - break; - } - } - tsBlockBuilder.declarePosition(); - } - TsBlock tsBlock = tsBlockBuilder.build(); - if (tsBlock == null) { - return null; - } else { - return serde.serialize(tsBlock); - } - } - - @SuppressWarnings( - "squid:S2095") // ignore Use try-with-resources or close this "Statement" in a "finally" - // clause @Override - public ResultSet getClientInfoProperties() throws SQLException { - Statement stmt = this.connection.createStatement(); - ResultSet rs; - try { - rs = stmt.executeQuery(SHOW_DATABASES_SQL); - } catch (SQLException e) { - stmt.close(); - throw e; - } - Field[] fields = new Field[4]; - fields[0] = new Field("", "NAME", "TEXT"); - fields[1] = new Field("", "MAX_LEN", INT32); - fields[2] = new Field("", "DEFAULT_VALUE", INT32); - fields[3] = new Field("", "DESCRIPTION", "TEXT"); - List tsDataTypeList = - Arrays.asList(TSDataType.TEXT, TSDataType.INT32, TSDataType.INT32, TSDataType.TEXT); - List values = Arrays.asList("fetch_size", 10, 10, ""); - - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List> valuesList = new ArrayList<>(); - - valuesList.add(values); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error("convert tsBlock error when get client info properties: {}", e.getMessage()); - } finally { - close(rs, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); + public String getTimeDateFunctions() throws SQLException { + return "COUNT_TIME,MAX_TIME,MIN_TIME,TIME_DIFFERENCE,TIME_DURATION"; } - @SuppressWarnings({ - "squid:S6541", - "squid:S3776", - "squid:S2095" - }) // ignore Cognitive Complexity of methods should not be too high - // ignore Methods should not perform too many tasks (aka Brain method) - // ignore Use try-with-resources or close this "Statement" in a "finally" @Override - public ResultSet getColumnPrivileges( - String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + public ResultSet getTables( + String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { Statement stmt = this.connection.createStatement(); - String sql = "SHOW DATABASES"; - if (catalog != null && catalog.length() > 0) { + String sql = "SHOW DEVICES"; + String database = ""; + if (catalog != null && !catalog.isEmpty()) { if (catalog.contains("%")) { catalog = catalog.replace("%", "*"); } + database = catalog; sql = sql + " " + catalog; - } else if (schemaPattern != null && schemaPattern.length() > 0) { + } else if (schemaPattern != null && !schemaPattern.isEmpty()) { if (schemaPattern.contains("%")) { schemaPattern = schemaPattern.replace("%", "*"); } + database = schemaPattern; sql = sql + " " + schemaPattern; } - if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + if (((catalog != null && !catalog.isEmpty()) + || schemaPattern != null && !schemaPattern.isEmpty()) && tableNamePattern != null - && tableNamePattern.length() > 0) { + && !tableNamePattern.isEmpty()) { if (tableNamePattern.contains("%")) { - tableNamePattern = tableNamePattern.replace("%", "*"); + tableNamePattern = tableNamePattern.replace("%", "**"); } sql = sql + "." + tableNamePattern; } - - if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) - && tableNamePattern != null - && tableNamePattern.length() > 0 - && columnNamePattern != null - && columnNamePattern.length() > 0) { - sql = sql + "." + columnNamePattern; - } + LOGGER.info("Get tables: sql: {}", sql); ResultSet rs; try { rs = stmt.executeQuery(sql); @@ -950,15 +302,18 @@ public ResultSet getColumnPrivileges( stmt.close(); throw e; } - Field[] fields = new Field[8]; + Field[] fields = new Field[10]; fields[0] = new Field("", TABLE_CAT, "TEXT"); fields[1] = new Field("", TABLE_SCHEM, "TEXT"); fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", COLUMN_NAME, "TEXT"); - fields[4] = new Field("", "GRANTOR", "TEXT"); - fields[5] = new Field("", "GRANTEE", "TEXT"); - fields[6] = new Field("", "PRIVILEGE", "TEXT"); - fields[7] = new Field("", "IS_GRANTABLE", "TEXT"); + fields[3] = new Field("", TABLE_TYPE, "TEXT"); + fields[4] = new Field("", REMARKS, "TEXT"); + fields[5] = new Field("", TYPE_CAT, "TEXT"); + fields[6] = new Field("", TYPE_SCHEM, "TEXT"); + fields[7] = new Field("", TYPE_NAME, "TEXT"); + fields[8] = new Field("", "SELF_REFERENCING_COL_NAME", "TEXT"); + fields[9] = new Field("", "REF_GENERATION", "TEXT"); + List tsDataTypeList = Arrays.asList( TSDataType.TEXT, @@ -968,6 +323,8 @@ public ResultSet getColumnPrivileges( TSDataType.TEXT, TSDataType.TEXT, TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, TSDataType.TEXT); List columnNameList = new ArrayList<>(); @@ -980,261 +337,136 @@ public ResultSet getColumnPrivileges( columnTypeList.add(fields[i].getSqlType()); columnNameIndex.put(fields[i].getName(), i); } + boolean hasResultSet = false; while (rs.next()) { - List valuesInRow = new ArrayList<>(); + hasResultSet = true; + List valueInRow = new ArrayList<>(); + String res = rs.getString(1); + for (int i = 0; i < fields.length; i++) { - if (i < 4) { - valuesInRow.add(rs.getString(1)); - } else if (i == 5) { - valuesInRow.add(getUserName()); - } else if (i == 6) { - valuesInRow.add(""); - } else if (i == 7) { - valuesInRow.add("NO"); + if (i < 2) { + valueInRow.add(""); + } else if (i == 2) { + int beginIndex = database.length() + 1; + if (StringUtils.isEmpty(database)) { + beginIndex = 0; + } + valueInRow.add(res.substring(beginIndex)); + } else if (i == 3) { + valueInRow.add("TABLE"); } else { - valuesInRow.add(""); + valueInRow.add(""); } } - valuesList.add(valuesInRow); + valuesList.add(valueInRow); } ByteBuffer tsBlock = null; try { tsBlock = convertTsBlock(valuesList, tsDataTypeList); } catch (IOException e) { - LOGGER.error("convert tsBlock error when get column privileges: {}", e.getMessage()); + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); } finally { close(rs, stmt); } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - @Override - public Connection getConnection() { - return connection; + return hasResultSet + ? new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId) + : null; } @Override - public ResultSet getCrossReference( - String arg0, String arg1, String arg2, String arg3, String arg4, String arg5) + public ResultSet getColumns( + String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[14]; - fields[0] = new Field("", PKTABLE_CAT, "TEXT"); - fields[1] = new Field("", PKTABLE_SCHEM, "TEXT"); - fields[2] = new Field("", PKTABLE_NAME, "TEXT"); - fields[3] = new Field("", PKCOLUMN_NAME, "TEXT"); - fields[4] = new Field("", FKTABLE_CAT, "TEXT"); - fields[5] = new Field("", FKTABLE_SCHEM, "TEXT"); - fields[6] = new Field("", FKTABLE_NAME, "TEXT"); - fields[7] = new Field("", FKCOLUMN_NAME, "TEXT"); - fields[8] = new Field("", KEY_SEQ, "TEXT"); - fields[9] = new Field("", "UPDATE_RULE ", "TEXT"); - fields[10] = new Field("", DELETE_RULE, "TEXT"); - fields[11] = new Field("", FK_NAME, "TEXT"); - fields[12] = new Field("", PK_NAME, "TEXT"); - fields[13] = new Field("", DEFERRABILITY, "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get cross reference error: {}", e.getMessage()); - } finally { - close(null, stmt); + Statement stmt = this.connection.createStatement(); + if (this.connection.getCatalog().equals(catalog)) { + catalog = null; } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - @Override - public int getDatabaseMajorVersion() { - int majorVersion = 0; - try { - String version = client.getProperties().getVersion(); - String[] versions = version.split("."); - if (versions.length >= 2) { - majorVersion = Integer.valueOf(versions[0]); + String sql = "SHOW TIMESERIES"; + if (org.apache.commons.lang3.StringUtils.isNotEmpty(catalog)) { + if (catalog.contains("%")) { + catalog = catalog.replace("%", "*"); } - } catch (TException e) { - LOGGER.error("get database major version error: {}", e.getMessage()); + sql = sql + " " + catalog; + } else if (org.apache.commons.lang3.StringUtils.isNotEmpty(schemaPattern)) { + if (schemaPattern.contains("%")) { + schemaPattern = schemaPattern.replace("%", "*"); + } + sql = sql + " " + schemaPattern; } - return majorVersion; - } - - @Override - public int getDatabaseMinorVersion() { - int minorVersion = 0; - try { - String version = client.getProperties().getVersion(); - String[] versions = version.split("."); - if (versions.length >= 2) { - minorVersion = Integer.valueOf(versions[1]); + if ((org.apache.commons.lang3.StringUtils.isNotEmpty(catalog) + || org.apache.commons.lang3.StringUtils.isNotEmpty(schemaPattern)) + && org.apache.commons.lang3.StringUtils.isNotEmpty(tableNamePattern)) { + if (tableNamePattern.contains("%")) { + tableNamePattern = tableNamePattern.replace("%", "*"); } - } catch (TException e) { - LOGGER.error("get database minor version error: {}", e.getMessage()); + sql = sql + "." + tableNamePattern; } - return minorVersion; - } - - @Override - public String getDatabaseProductName() { - return Constant.GLOBAL_DB_NAME; - } - - @Override - public String getDatabaseProductVersion() { - return getDriverVersion(); - } - - @Override - public int getDefaultTransactionIsolation() { - return 0; - } - - @Override - public int getDriverMajorVersion() { - return 4; - } - - @Override - public int getDriverMinorVersion() { - return 3; - } - - @Override - public String getDriverName() { - return org.apache.iotdb.jdbc.IoTDBDriver.class.getName(); - } - @Override - public String getDriverVersion() { - return DATABASE_VERSION; - } - - @Override - public ResultSet getExportedKeys(String catalog, String schema, final String table) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[14]; - fields[0] = new Field("", PKTABLE_CAT, "TEXT"); - fields[1] = new Field("", PKTABLE_SCHEM, INT32); - fields[2] = new Field("", PKTABLE_NAME, "TEXT"); - fields[3] = new Field("", PKCOLUMN_NAME, "TEXT"); - fields[4] = new Field("", FKTABLE_CAT, "TEXT"); - fields[5] = new Field("", FKTABLE_SCHEM, "TEXT"); - fields[6] = new Field("", FKTABLE_NAME, "TEXT"); - fields[7] = new Field("", FKCOLUMN_NAME, "TEXT"); - fields[8] = new Field("", KEY_SEQ, INT32); - fields[9] = new Field("", "UPDATE_RULE", INT32); - fields[10] = new Field("", DELETE_RULE, INT32); - fields[11] = new Field("", FK_NAME, "TEXT"); - fields[12] = new Field("", PK_NAME, "TEXT"); - fields[13] = new Field("", DEFERRABILITY, INT32); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); + if ((org.apache.commons.lang3.StringUtils.isNotEmpty(catalog) + || org.apache.commons.lang3.StringUtils.isNotEmpty(schemaPattern)) + && org.apache.commons.lang3.StringUtils.isNotEmpty(tableNamePattern) + && org.apache.commons.lang3.StringUtils.isNotEmpty(columnNamePattern)) { + if (columnNamePattern.contains("%")) { + columnNamePattern = columnNamePattern.replace("%", "*"); } - } catch (Exception e) { - LOGGER.error("get exported keys error: {}", e.getMessage()); - } finally { - close(null, stmt); + sql = sql + "." + columnNamePattern; } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public String getExtraNameCharacters() { - return ""; - } - @SuppressWarnings( - "squid:S2095") // ignore Use try-with-resources or close this "Statement" in a "finally" - // clause - @Override - public ResultSet getFunctionColumns( - String catalog, - String schemaPattern, - java.lang.String functionNamePattern, - java.lang.String columnNamePattern) - throws SQLException { - Statement stmt = connection.createStatement(); + if (org.apache.commons.lang3.StringUtils.isEmpty(catalog) + && org.apache.commons.lang3.StringUtils.isEmpty(schemaPattern) + && StringUtils.isNotEmpty(tableNamePattern)) { + sql = sql + " " + tableNamePattern + ".*"; + } ResultSet rs; try { - rs = stmt.executeQuery(SHOW_FUNCTIONS); + rs = stmt.executeQuery(sql); } catch (SQLException e) { stmt.close(); throw e; } - Field[] fields = new Field[17]; - fields[0] = new Field("", "FUNCTION_CAT ", "TEXT"); - fields[1] = new Field("", "FUNCTION_SCHEM", "TEXT"); - fields[2] = new Field("", "FUNCTION_NAME", "TEXT"); + Field[] fields = new Field[24]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); fields[3] = new Field("", COLUMN_NAME, "TEXT"); - fields[4] = new Field("", "COLUMN_TYPE", INT32); - fields[5] = new Field("", DATA_TYPE, INT32); - fields[6] = new Field("", TYPE_NAME, "TEXT"); - fields[7] = new Field("", PRECISION, INT32); - fields[8] = new Field("", "LENGTH", INT32); - fields[9] = new Field("", "SCALE", INT32); - fields[10] = new Field("", "RADIX", INT32); - fields[11] = new Field("", NULLABLE, INT32); - fields[12] = new Field("", REMARKS, "TEXT"); - fields[13] = new Field("", CHAR_OCTET_LENGTH, INT32); - fields[14] = new Field("", ORDINAL_POSITION, INT32); - fields[15] = new Field("", IS_NULLABLE, "TEXT"); - fields[16] = new Field("", SPECIFIC_NAME, "TEXT"); + fields[4] = new Field("", DATA_TYPE, INT32); + fields[5] = new Field("", TYPE_NAME, "TEXT"); + fields[6] = new Field("", COLUMN_SIZE, INT32); + fields[7] = new Field("", BUFFER_LENGTH, INT32); + fields[8] = new Field("", DECIMAL_DIGITS, INT32); + fields[9] = new Field("", NUM_PREC_RADIX, INT32); + fields[10] = new Field("", NULLABLE, INT32); + fields[11] = new Field("", REMARKS, "TEXT"); + fields[12] = new Field("", COLUMN_DEF, "TEXT"); + fields[13] = new Field("", SQL_DATA_TYPE, INT32); + fields[14] = new Field("", SQL_DATETIME_SUB, INT32); + fields[15] = new Field("", CHAR_OCTET_LENGTH, INT32); + fields[16] = new Field("", ORDINAL_POSITION, INT32); + fields[17] = new Field("", IS_NULLABLE, "TEXT"); + fields[18] = new Field("", SCOPE_CATALOG, "TEXT"); + fields[19] = new Field("", SCOPE_SCHEMA, "TEXT"); + fields[20] = new Field("", SCOPE_TABLE, "TEXT"); + fields[21] = new Field("", SOURCE_DATA_TYPE, INT32); + fields[22] = new Field("", IS_AUTOINCREMENT, "TEXT"); + fields[23] = new Field("", IS_GENERATEDCOLUMN, "TEXT"); + List tsDataTypeList = Arrays.asList( TSDataType.TEXT, @@ -1242,7 +474,6 @@ public ResultSet getFunctionColumns( TSDataType.TEXT, TSDataType.TEXT, TSDataType.INT32, - TSDataType.INT32, TSDataType.TEXT, TSDataType.INT32, TSDataType.INT32, @@ -1250,28 +481,83 @@ public ResultSet getFunctionColumns( TSDataType.INT32, TSDataType.INT32, TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, TSDataType.INT32, TSDataType.TEXT, TSDataType.TEXT); - List columnNameList = new ArrayList<>(); List columnTypeList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); Map columnNameIndex = new HashMap<>(); - + List> valuesList = new ArrayList<>(); for (int i = 0; i < fields.length; i++) { columnNameList.add(fields[i].getName()); columnTypeList.add(fields[i].getSqlType()); columnNameIndex.put(fields[i].getName(), i); } + while (rs.next()) { List valuesInRow = new ArrayList<>(); + String res = rs.getString(1); + String[] splitRes = res.split("\\."); for (int i = 0; i < fields.length; i++) { - if (i == 2) { - valuesInRow.add(rs.getString(1)); - } else if (INT32.equals(fields[i].getSqlType())) { + if (i == 0) { + valuesInRow.add(""); + } else if (i == 1) { + valuesInRow.add(schemaPattern); + } else if (i == 2) { + valuesInRow.add( + res.substring(0, res.length() - splitRes[splitRes.length - 1].length() - 1)); + } else if (i == 3) { + // column name + valuesInRow.add(splitRes[splitRes.length - 1]); + } else if (i == 4) { + valuesInRow.add(getSQLType(rs.getString(4))); + } else if (i == 5) { + valuesInRow.add(rs.getString(4)); + } else if (i == 6) { + valuesInRow.add(getTypePrecision(fields[i].getSqlType())); + } else if (i == 7) { + valuesInRow.add(0); + } else if (i == 8) { + valuesInRow.add(getTypeScale(fields[i].getSqlType())); + } else if (i == 9) { + valuesInRow.add(10); + } else if (i == 10) { + valuesInRow.add(0); + } else if (i == 11) { + valuesInRow.add(""); + } else if (i == 12) { + valuesInRow.add(""); + } else if (i == 13) { + valuesInRow.add(0); + } else if (i == 14) { + valuesInRow.add(0); + } else if (i == 15) { + valuesInRow.add(getTypePrecision(fields[i].getSqlType())); + } else if (i == 16) { + valuesInRow.add(1); + } else if (i == 17) { + valuesInRow.add("NO"); + } else if (i == 18) { + valuesInRow.add(""); + } else if (i == 19) { + valuesInRow.add(""); + } else if (i == 20) { + valuesInRow.add(""); + } else if (i == 21) { valuesInRow.add(0); + } else if (i == 22) { + valuesInRow.add("NO"); + } else if (i == 23) { + valuesInRow.add("NO"); } else { valuesInRow.add(""); } @@ -1283,10 +569,11 @@ public ResultSet getFunctionColumns( try { tsBlock = convertTsBlock(valuesList, tsDataTypeList); } catch (IOException e) { - LOGGER.error("convert tsBlock error when get function columns: {}", e.getMessage()); + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); } finally { close(rs, stmt); } + return new IoTDBJDBCResultSet( stmt, columnNameList, @@ -1304,27 +591,12 @@ public ResultSet getFunctionColumns( zoneId); } - @SuppressWarnings( - "squid:S2095") // ignore Use try-with-resources or close this "Statement" in a "finally" - // clause @Override - public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) - throws SQLException { + public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { Statement stmt = connection.createStatement(); - ResultSet rs; - try { - rs = stmt.executeQuery(SHOW_FUNCTIONS); - } catch (SQLException e) { - stmt.close(); - throw e; - } - Field[] fields = new Field[6]; - fields[0] = new Field("", "FUNCTION_CAT ", "TEXT"); - fields[1] = new Field("", "FUNCTION_SCHEM", "TEXT"); - fields[2] = new Field("", "FUNCTION_NAME", "TEXT"); - fields[3] = new Field("", REMARKS, "TEXT"); - fields[4] = new Field("", "FUNCTION_TYPE", INT32); - fields[5] = new Field("", SPECIFIC_NAME, "TEXT"); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); List tsDataTypeList = Arrays.asList( TSDataType.TEXT, @@ -1334,335 +606,11 @@ public ResultSet getFunctions(String catalog, String schemaPattern, String funct TSDataType.INT32, TSDataType.TEXT); - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List> valuesList = new ArrayList<>(); - - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - while (rs.next()) { - List valueInRow = new ArrayList<>(); - for (int i = 0; i < fields.length; i++) { - if (i == 2) { - valueInRow.add(rs.getString(1)); - } else if (i == 4) { - valueInRow.add(0); - } else { - valueInRow.add(""); - } - } - valuesList.add(valueInRow); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error("convert tsBlock error when get functions: {}", e.getMessage()); - } finally { - close(rs, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public String getIdentifierQuoteString() { - return "\' or \""; - } - - @Override - public ResultSet getImportedKeys(String arg0, String arg1, String arg2) throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[14]; - fields[0] = new Field("", PKTABLE_CAT, "TEXT"); - fields[1] = new Field("", PKTABLE_SCHEM, INT32); - fields[2] = new Field("", PKTABLE_NAME, "TEXT"); - fields[3] = new Field("", PKCOLUMN_NAME, "TEXT"); - fields[4] = new Field("", FKTABLE_CAT, "TEXT"); - fields[5] = new Field("", FKTABLE_SCHEM, "TEXT"); - fields[6] = new Field("", FKTABLE_NAME, "TEXT"); - fields[7] = new Field("", FKCOLUMN_NAME, "TEXT"); - fields[8] = new Field("", KEY_SEQ, INT32); - fields[9] = new Field("", "UPDATE_RULE", INT32); - fields[10] = new Field("", DELETE_RULE, INT32); - fields[11] = new Field("", FK_NAME, "TEXT"); - fields[12] = new Field("", PK_NAME, "TEXT"); - fields[13] = new Field("", DEFERRABILITY, INT32); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get import keys error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public ResultSet getIndexInfo(String arg0, String arg1, String arg2, boolean arg3, boolean arg4) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[14]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", "NON_UNIQUE", "TEXT"); - fields[4] = new Field("", "INDEX_QUALIFIER", "TEXT"); - fields[5] = new Field("", "INDEX_NAME", "TEXT"); - fields[6] = new Field("", "TYPE", "TEXT"); - fields[7] = new Field("", ORDINAL_POSITION, "TEXT"); - fields[8] = new Field("", COLUMN_NAME, "TEXT"); - fields[9] = new Field("", "ASC_OR_DESC", "TEXT"); - fields[10] = new Field("", "CARDINALITY", "TEXT"); - fields[11] = new Field("", "PAGES", "TEXT"); - fields[12] = new Field("", PK_NAME, "TEXT"); - fields[13] = new Field("", "FILTER_CONDITION", "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get index info error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public int getJDBCMajorVersion() { - return 4; - } - - @Override - public int getJDBCMinorVersion() { - return 3; - } - - @Override - public int getMaxBinaryLiteralLength() { - return Integer.MAX_VALUE; - } - - /** Although there is no limit, it is not recommended */ - @Override - public int getMaxCatalogNameLength() { - return 1024; - } - - @Override - public int getMaxCharLiteralLength() { - return Integer.MAX_VALUE; - } - - /** Although there is no limit, it is not recommended */ - @Override - public int getMaxColumnNameLength() { - return 1024; - } - - @Override - public int getMaxColumnsInGroupBy() { - return 1; - } - - @Override - public int getMaxColumnsInIndex() { - return 0; - } - - @Override - public int getMaxColumnsInOrderBy() { - return 1; - } - - @Override - public int getMaxColumnsInSelect() { - return 0; - } - - @Override - public int getMaxColumnsInTable() { - return Integer.MAX_VALUE; - } - - @Override - public int getMaxConnections() { - int maxcount = 0; - try { - maxcount = client.getProperties().getMaxConcurrentClientNum(); - } catch (TException e) { - LOGGER.error("get max concurrentClientNUm error: {}", e.getMessage()); - } - return maxcount; - } - - @Override - public int getMaxCursorNameLength() { - return 0; - } - - @Override - public int getMaxIndexLength() { - return Integer.MAX_VALUE; - } - - @Override - public int getMaxProcedureNameLength() { - return 0; - } - - /** maxrowsize unlimited */ - @Override - public int getMaxRowSize() { - return 2147483639; - } - - /** Although there is no limit, it is not recommended */ - @Override - public int getMaxSchemaNameLength() { - return 1024; - } - - @Override - public int getMaxStatementLength() { - try { - return client.getProperties().getThriftMaxFrameSize(); - } catch (TException e) { - LOGGER.error("get max statement length error: {}", e.getMessage()); - } - return 0; - } - - @Override - public int getMaxStatements() { - return 0; - } - - /** Although there is no limit, it is not recommended */ - @Override - public int getMaxTableNameLength() { - return 1024; - } - - /** Although there is no limit, it is not recommended */ - @Override - public int getMaxTablesInSelect() { - return 1024; - } - - /** Although there is no limit, it is not recommended */ - @Override - public int getMaxUserNameLength() { - return 1024; - } - - @Override - public String getNumericFunctions() { - ResultSet resultSet = null; - Statement statement = null; - String result = ""; - try { - statement = connection.createStatement(); - StringBuilder str = new StringBuilder(""); - resultSet = statement.executeQuery(SHOW_FUNCTIONS); - List listfunction = Arrays.asList("MAX_TIME", "MIN_TIME", "TIME_DIFFERENCE", "NOW"); - while (resultSet.next()) { - if (listfunction.contains(resultSet.getString(1))) { - continue; - } - str.append(resultSet.getString(1)).append(","); - } - result = str.toString(); - if (result.length() > 0) { - result = result.substring(0, result.length() - 1); - } - } catch (Exception e) { - LOGGER.error("get numeric functions error: {}", e.getMessage()); - } finally { - close(resultSet, statement); - } - return result; - } - - @Override - public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { - Statement stmt = connection.createStatement(); - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT); - - String database = ""; - if (catalog != null) { - database = catalog; - } else if (schema != null) { - database = schema; + String database = ""; + if (catalog != null) { + database = catalog; + } else if (schema != null) { + database = schema; } Field[] fields = new Field[6]; @@ -1686,7 +634,7 @@ public ResultSet getPrimaryKeys(String catalog, String schema, String table) thr try { tsBlock = convertTsBlock(valuesList, tsDataTypeList); } catch (IOException e) { - LOGGER.error("get primary keys error: {}", e.getMessage()); + LOGGER.error("Get primary keys error: {}", e.getMessage()); } finally { close(null, stmt); } @@ -1708,1702 +656,18 @@ public ResultSet getPrimaryKeys(String catalog, String schema, String table) thr } @Override - public ResultSet getProcedureColumns(String arg0, String arg1, String arg2, String arg3) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - - Field[] fields = new Field[20]; - fields[0] = new Field("", "PROCEDURE_CAT", "TEXT"); - fields[1] = new Field("", "PROCEDURE_SCHEM", "TEXT"); - fields[2] = new Field("", "PROCEDURE_NAME", "TEXT"); - fields[3] = new Field("", COLUMN_NAME, "TEXT"); - fields[4] = new Field("", "COLUMN_TYPE", "TEXT"); - fields[5] = new Field("", DATA_TYPE, INT32); - fields[6] = new Field("", TYPE_NAME, "TEXT"); - fields[7] = new Field("", PRECISION, "TEXT"); - fields[8] = new Field("", "LENGTH", "TEXT"); - fields[9] = new Field("", "SCALE", "TEXT"); - fields[10] = new Field("", "RADIX", "TEXT"); - fields[11] = new Field("", NULLABLE, "TEXT"); - fields[12] = new Field("", REMARKS, "TEXT"); - fields[13] = new Field("", "COLUMN_DEF", "TEXT"); - fields[14] = new Field("", SQL_DATA_TYPE, INT32); - fields[15] = new Field("", SQL_DATETIME_SUB, "TEXT"); - fields[16] = new Field("", CHAR_OCTET_LENGTH, "TEXT"); - fields[17] = new Field("", ORDINAL_POSITION, "TEXT"); - fields[18] = new Field("", IS_NULLABLE, "TEXT"); - fields[19] = new Field("", SPECIFIC_NAME, "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get procedure columns error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); + public boolean supportsSchemasInDataManipulation() throws SQLException { + return true; } @Override - public String getProcedureTerm() { + public String getIdentifierQuoteString() throws SQLException { return ""; } @Override - public ResultSet getProcedures(String arg0, String arg1, String arg2) throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[6]; - fields[0] = new Field("", "PROCEDURE_CAT", "TEXT"); - fields[1] = new Field("", "PROCEDURE_SCHEM", "TEXT"); - fields[2] = new Field("", "PROCEDURE_NAME", "TEXT"); - fields[3] = new Field("", REMARKS, "TEXT"); - fields[4] = new Field("", "PROCEDURE_TYPE", "TEXT"); - fields[5] = new Field("", SPECIFIC_NAME, "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get procedures error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public ResultSet getPseudoColumns( - String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) - throws SQLException { - Statement stmt = connection.createStatement(); - Field[] fields = new Field[12]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", COLUMN_NAME, "TEXT"); - fields[4] = new Field("", DATA_TYPE, INT32); - fields[5] = new Field("", COLUMN_SIZE, INT32); - fields[6] = new Field("", DECIMAL_DIGITS, INT32); - fields[7] = new Field("", NUM_PREC_RADIX, INT32); - fields[8] = new Field("", "COLUMN_USAGE", "TEXT"); - fields[9] = new Field("", REMARKS, "TEXT"); - fields[10] = new Field("", CHAR_OCTET_LENGTH, INT32); - fields[11] = new Field("", IS_NULLABLE, "TEXT"); - List tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT); - - List value = - Arrays.asList( - catalog, catalog, tableNamePattern, "times", Types.BIGINT, 1, 0, 2, "", "", 13, "NO"); - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(Collections.singletonList(value), tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - 0, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public int getResultSetHoldability() { - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - @Override - public RowIdLifetime getRowIdLifetime() { - return RowIdLifetime.ROWID_UNSUPPORTED; - } - - @Override - public String getSQLKeywords() { - return sqlKeywordsThatArentSQL92; - } - - @Override - public int getSQLStateType() { - return 0; - } - - @Override - public String getSchemaTerm() { - return "stroge group"; - } - - @SuppressWarnings( - "squid:S2095") // ignore Use try-with-resources or close this "Statement" in a "finally" - // clause - @Override - public ResultSet getSchemas() throws SQLException { - Statement stmt = this.connection.createStatement(); - ResultSet rs; - try { - rs = stmt.executeQuery(SHOW_DATABASES_SQL); - } catch (SQLException e) { - stmt.close(); - throw e; - } - Field[] fields = new Field[2]; - fields[0] = new Field("", TABLE_SCHEM, "TEXT"); - fields[1] = new Field("", "TABLE_CATALOG", "TEXT"); - - List tsDataTypeList = Arrays.asList(TSDataType.TEXT, TSDataType.TEXT); - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List> valuesList = new ArrayList<>(); - - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - while (rs.next()) { - List valueInRow = new ArrayList<>(); - for (int i = 0; i < fields.length; i++) { - valueInRow.add(rs.getString(1)); - } - valuesList.add(valueInRow); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } finally { - close(rs, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { - return getSchemas(); - } - - @Override - public String getSearchStringEscape() { - return "\\"; - } - - @Override - public String getStringFunctions() { - return getSystemFunctions(); - } - - @Override - public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[4]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", "SUPERTABLE_NAME", "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get super tables error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[6]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", "SUPERTYPE_CAT", "TEXT"); - fields[4] = new Field("", "SUPERTYPE_SCHEM", "TEXT"); - fields[5] = new Field("", "SUPERTYPE_NAME", "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get super types error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public String getSystemFunctions() { - String result = ""; - Statement statement = null; - ResultSet resultSet = null; - try { - statement = connection.createStatement(); - StringBuilder str = new StringBuilder(""); - resultSet = statement.executeQuery(SHOW_FUNCTIONS); - while (resultSet.next()) { - str.append(resultSet.getString(1)).append(","); - } - result = str.toString(); - if (result.length() > 0) { - result = result.substring(0, result.length() - 1); - } - } catch (Exception ex) { - LOGGER.error("get system functions error: {}", ex.getMessage()); - } finally { - close(resultSet, statement); - } - return result; - } - - @SuppressWarnings({ - "squid:S6541", - "squid:S3776", - "squid:S2095" - }) // ignore Cognitive Complexity of methods should not be too high - // ignore Methods should not perform too many tasks (aka Brain method) - // ignore Use try-with-resources or close this "Statement" in a "finally" clause - @Override - public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) - throws SQLException { - Statement stmt = this.connection.createStatement(); - - String sql = "SHOW DATABASES"; - if (catalog != null && catalog.length() > 0) { - if (catalog.contains("%")) { - catalog = catalog.replace("%", "*"); - } - sql = sql + " " + catalog; - } else if (schemaPattern != null && schemaPattern.length() > 0) { - if (schemaPattern.contains("%")) { - schemaPattern = schemaPattern.replace("%", "*"); - } - sql = sql + " " + schemaPattern; - } - if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) - && tableNamePattern != null - && tableNamePattern.length() > 0) { - if (tableNamePattern.contains("%")) { - tableNamePattern = tableNamePattern.replace("%", "*"); - } - sql = sql + "." + tableNamePattern; - } - ResultSet rs; - try { - rs = stmt.executeQuery(sql); - } catch (SQLException e) { - stmt.close(); - throw e; - } - Field[] fields = new Field[8]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", COLUMN_NAME, "TEXT"); - fields[4] = new Field("", "GRANTOR", "TEXT"); - fields[5] = new Field("", "GRANTEE", "TEXT"); - fields[6] = new Field("", "PRIVILEGE", "TEXT"); - fields[7] = new Field("", "IS_GRANTABLE", "TEXT"); - List tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT); - - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List> valuesList = new ArrayList<>(); - - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - while (rs.next()) { - List valueInRow = new ArrayList<>(); - for (int i = 0; i < fields.length; i++) { - if (i < 4) { - valueInRow.add(rs.getString(1)); - } else if (i == 5) { - valueInRow.add(getUserName()); - } else if (i == 6) { - valueInRow.add(""); - } else if (i == 7) { - valueInRow.add("NO"); - } else { - valueInRow.add(""); - } - } - valuesList.add(valueInRow); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } finally { - close(rs, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public ResultSet getTableTypes() throws SQLException { - Statement stmt = this.connection.createStatement(); - - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List tsDataTypeList = new ArrayList<>(); - List value = new ArrayList<>(); - - tsDataTypeList.add(TSDataType.TEXT); - value.add("table"); - columnNameList.add(TABLE_TYPE); - columnTypeList.add("TEXT"); - columnNameIndex.put(TABLE_TYPE, 0); - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(Collections.singletonList(value), tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @SuppressWarnings({ - "squid:S6541", - "squid:S3776", - "squid:S2095" - }) // ignore Cognitive Complexity of methods should not be too high - // ignore Methods should not perform too many tasks (aka Brain method) - // ignore Use try-with-resources or close this "Statement" in a "finally" clause - @Override - public ResultSet getColumns( - String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) - throws SQLException { - Statement stmt = this.connection.createStatement(); - - if (this.connection.getCatalog().equals(catalog)) { - catalog = null; - } - - String sql = "SHOW TIMESERIES"; - if (StringUtils.isNotEmpty(catalog)) { - if (catalog.contains("%")) { - catalog = catalog.replace("%", "*"); - } - sql = sql + " " + catalog; - } else if (StringUtils.isNotEmpty(schemaPattern)) { - if (schemaPattern.contains("%")) { - schemaPattern = schemaPattern.replace("%", "*"); - } - sql = sql + " " + schemaPattern; - } - if ((StringUtils.isNotEmpty(catalog) || StringUtils.isNotEmpty(schemaPattern)) - && StringUtils.isNotEmpty(tableNamePattern)) { - if (tableNamePattern.contains("%")) { - tableNamePattern = tableNamePattern.replace("%", "*"); - } - sql = sql + "." + tableNamePattern; - } - - if ((StringUtils.isNotEmpty(catalog) || StringUtils.isNotEmpty(schemaPattern)) - && StringUtils.isNotEmpty(tableNamePattern) - && StringUtils.isNotEmpty(columnNamePattern)) { - if (columnNamePattern.contains("%")) { - columnNamePattern = columnNamePattern.replace("%", "*"); - } - sql = sql + "." + columnNamePattern; - } - - if (StringUtils.isEmpty(catalog) - && StringUtils.isEmpty(schemaPattern) - && StringUtils.isNotEmpty(tableNamePattern)) { - sql = sql + " " + tableNamePattern + ".*"; - } - ResultSet rs; - try { - rs = stmt.executeQuery(sql); - } catch (SQLException e) { - stmt.close(); - throw e; - } - Field[] fields = new Field[24]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", COLUMN_NAME, "TEXT"); - fields[4] = new Field("", DATA_TYPE, INT32); - fields[5] = new Field("", TYPE_NAME, "TEXT"); - fields[6] = new Field("", COLUMN_SIZE, INT32); - fields[7] = new Field("", BUFFER_LENGTH, INT32); - fields[8] = new Field("", DECIMAL_DIGITS, INT32); - fields[9] = new Field("", NUM_PREC_RADIX, INT32); - fields[10] = new Field("", NULLABLE, INT32); - fields[11] = new Field("", REMARKS, "TEXT"); - fields[12] = new Field("", "COLUMN_DEF", "TEXT"); - fields[13] = new Field("", SQL_DATA_TYPE, INT32); - fields[14] = new Field("", SQL_DATETIME_SUB, INT32); - fields[15] = new Field("", CHAR_OCTET_LENGTH, INT32); - fields[16] = new Field("", ORDINAL_POSITION, INT32); - fields[17] = new Field("", IS_NULLABLE, "TEXT"); - fields[18] = new Field("", "SCOPE_CATALOG", "TEXT"); - fields[19] = new Field("", "SCOPE_SCHEMA", "TEXT"); - fields[20] = new Field("", "SCOPE_TABLE", "TEXT"); - fields[21] = new Field("", "SOURCE_DATA_TYPE", INT32); - fields[22] = new Field("", "IS_AUTOINCREMENT", "TEXT"); - fields[23] = new Field("", "IS_GENERATEDCOLUMN", "TEXT"); - - List tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT); - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List> valuesList = new ArrayList<>(); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - while (rs.next()) { - List valuesInRow = new ArrayList<>(); - String res = rs.getString(1); - String[] splitRes = res.split("\\."); - for (int i = 0; i < fields.length; i++) { - if (i <= 1) { - valuesInRow.add(" "); - } else if (i == 2) { - valuesInRow.add( - res.substring(0, res.length() - splitRes[splitRes.length - 1].length() - 1)); - } else if (i == 3) { - // column name - valuesInRow.add(splitRes[splitRes.length - 1]); - } else if (i == 4) { - valuesInRow.add(getSQLType(rs.getString(4))); - } else if (i == 6) { - valuesInRow.add(getTypePrecision(fields[i].getSqlType())); - } else if (i == 7) { - valuesInRow.add(0); - } else if (i == 8) { - valuesInRow.add(getTypeScale(fields[i].getSqlType())); - } else if (i == 9) { - valuesInRow.add(10); - } else if (i == 10) { - valuesInRow.add(0); - } else if (i == 11) { - valuesInRow.add(""); - } else if (i == 12) { - valuesInRow.add(""); - } else if (i == 13) { - valuesInRow.add(0); - } else if (i == 14) { - valuesInRow.add(0); - } else if (i == 15) { - valuesInRow.add(getTypePrecision(fields[i].getSqlType())); - } else if (i == 16) { - valuesInRow.add(1); - } else if (i == 17) { - valuesInRow.add("NO"); - } else if (i == 18) { - valuesInRow.add(""); - } else if (i == 19) { - valuesInRow.add(""); - } else if (i == 20) { - valuesInRow.add(""); - } else if (i == 21) { - valuesInRow.add(0); - } else if (i == 22) { - valuesInRow.add("NO"); - } else if (i == 23) { - valuesInRow.add("NO"); - } else { - valuesInRow.add(""); - } - } - valuesList.add(valuesInRow); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } finally { - close(rs, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - private void close(ResultSet rs, Statement stmt) { - - try { - if (rs != null) { - rs.close(); - } - } catch (Exception ex) { - rs = null; - } - try { - if (stmt != null) { - stmt.close(); - } - } catch (Exception ex) { - stmt = null; - } - } - - public int getTypeScale(String columnType) { - switch (columnType.toUpperCase()) { - case BOOLEAN: - case INT32: - case INT64: - case "TEXT": - return 0; - case FLOAT: - return 6; - case DOUBLE: - return 15; - default: - break; - } - return 0; - } - - private int getSQLType(String columnType) { - switch (columnType.toUpperCase()) { - case BOOLEAN: - return Types.BOOLEAN; - case INT32: - return Types.INTEGER; - case INT64: - return Types.BIGINT; - case FLOAT: - return Types.FLOAT; - case DOUBLE: - return Types.DOUBLE; - case "TEXT": - return Types.LONGVARCHAR; - default: - break; - } - return 0; - } - - private int getTypePrecision(String columnType) { - // BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, - switch (columnType.toUpperCase()) { - case BOOLEAN: - return 1; - case INT32: - return 10; - case INT64: - return 19; - case FLOAT: - return 38; - case DOUBLE: - return 308; - case "TEXT": - return Integer.MAX_VALUE; - default: - break; - } - return 0; - } - - @SuppressWarnings({ - "squid:S6541", - "squid:S3776", - "squid:S2095" - }) // ignore Cognitive Complexity of methods should not be too high - // ignore Methods should not perform too many tasks (aka Brain method) - // ignore Use try-with-resources or close this "Statement" in a "finally" clause - @Override - public ResultSet getTables( - String catalog, String schemaPattern, String tableNamePattern, String[] types) - throws SQLException { - Statement stmt = this.connection.createStatement(); - - String sql = "SHOW DEVICES"; - String database = ""; - if (catalog != null && catalog.length() > 0) { - if (catalog.contains("%")) { - catalog = catalog.replace("%", "*"); - } - database = catalog; - sql = sql + " " + catalog; - } else if (schemaPattern != null && schemaPattern.length() > 0) { - if (schemaPattern.contains("%")) { - schemaPattern = schemaPattern.replace("%", "*"); - } - database = schemaPattern; - sql = sql + " " + schemaPattern; - } - if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) - && tableNamePattern != null - && tableNamePattern.length() > 0) { - if (tableNamePattern.contains("%")) { - tableNamePattern = tableNamePattern.replace("%", "**"); - } - sql = sql + "." + tableNamePattern; - } - ResultSet rs; - try { - rs = stmt.executeQuery(sql); - } catch (SQLException e) { - stmt.close(); - throw e; - } - Field[] fields = new Field[10]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", TABLE_TYPE, "TEXT"); - fields[4] = new Field("", REMARKS, "TEXT"); - fields[5] = new Field("", TYPE_CAT, "TEXT"); - fields[6] = new Field("", "TYPE_SCHEM", "TEXT"); - fields[7] = new Field("", TYPE_NAME, "TEXT"); - fields[8] = new Field("", "SELF_REFERENCING_COL_NAME", "TEXT"); - fields[9] = new Field("", "REF_GENERATION", "TEXT"); - - List tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT); - - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - List> valuesList = new ArrayList<>(); - - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - while (rs.next()) { - List valueInRow = new ArrayList<>(); - String res = rs.getString(1); - - for (int i = 0; i < fields.length; i++) { - if (i < 2) { - valueInRow.add(""); - } else if (i == 2) { - int beginIndex = database.length() + 1; - if (StringUtils.isEmpty(database)) { - beginIndex = 0; - } - valueInRow.add(res.substring(beginIndex)); - } else if (i == 3) { - valueInRow.add("TABLE"); - } else { - valueInRow.add(""); - } - } - valuesList.add(valueInRow); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public String getTimeDateFunctions() { - return "MAX_TIME,MIN_TIME,TIME_DIFFERENCE,NOW"; - } - - @Override - public ResultSet getTypeInfo() throws SQLException { - Statement stmt = connection.createStatement(); - Field[] fields = new Field[18]; - fields[0] = new Field("", TYPE_NAME, "TEXT"); - fields[1] = new Field("", DATA_TYPE, INT32); - fields[2] = new Field("", PRECISION, INT32); - fields[3] = new Field("", "LITERAL_PREFIX", "TEXT"); - fields[4] = new Field("", "LITERAL_SUFFIX", "TEXT"); - fields[5] = new Field("", "CREATE_PARAMS", "TEXT"); - fields[6] = new Field("", NULLABLE, INT32); - fields[7] = new Field("", "CASE_SENSITIVE", BOOLEAN); - fields[8] = new Field("", "SEARCHABLE", "TEXT"); - fields[9] = new Field("", "UNSIGNED_ATTRIBUTE", BOOLEAN); - fields[10] = new Field("", "FIXED_PREC_SCALE", BOOLEAN); - fields[11] = new Field("", "AUTO_INCREMENT", BOOLEAN); - fields[12] = new Field("", "LOCAL_TYPE_NAME", "TEXT"); - fields[13] = new Field("", "MINIMUM_SCALE", INT32); - fields[14] = new Field("", "MAXIMUM_SCALE", INT32); - fields[15] = new Field("", SQL_DATA_TYPE, INT32); - fields[16] = new Field("", SQL_DATETIME_SUB, INT32); - fields[17] = new Field("", NUM_PREC_RADIX, INT32); - List tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.BOOLEAN, - TSDataType.TEXT, - TSDataType.BOOLEAN, - TSDataType.BOOLEAN, - TSDataType.BOOLEAN, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32); - List listValSub1 = - Arrays.asList( - INT32, - Types.INTEGER, - 10, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List listValSub2 = - Arrays.asList( - INT64, - Types.BIGINT, - 19, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List listValSub3 = - Arrays.asList( - BOOLEAN, - Types.BOOLEAN, - 1, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List listValSub4 = - Arrays.asList( - FLOAT, - Types.FLOAT, - 38, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List listValSub5 = - Arrays.asList( - DOUBLE, - Types.DOUBLE, - 308, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List listValSub6 = - Arrays.asList( - "TEXT", - Types.LONGVARCHAR, - 64, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List> valuesList = - Arrays.asList(listValSub1, listValSub2, listValSub3, listValSub4, listValSub5, listValSub6); - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - - ByteBuffer tsBlock = null; - try { - tsBlock = convertTsBlock(valuesList, tsDataTypeList); - } catch (IOException e) { - LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - true, - client, - null, - -1, - sessionId, - Collections.singletonList(tsBlock), - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public ResultSet getUDTs( - String catalog, String schemaPattern, String typeNamePattern, int[] types) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[7]; - fields[0] = new Field("", TABLE_CAT, "TEXT"); - fields[1] = new Field("", TABLE_SCHEM, "TEXT"); - fields[2] = new Field("", TABLE_NAME, "TEXT"); - fields[3] = new Field("", "CLASS_NAME", "TEXT"); - fields[4] = new Field("", DATA_TYPE, INT32); - fields[5] = new Field("", REMARKS, "TEXT"); - fields[6] = new Field("", "BASE_TYPE", "TEXT"); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get UDTS error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public String getURL() { - // TODO: Return the URL for this DBMS or null if it cannot be generated - return this.connection.getUrl(); - } - - @Override - public String getUserName() throws SQLException { - return connection.getUserName(); - } - - @Override - public ResultSet getVersionColumns(String catalog, String schema, String table) - throws SQLException { - List columnNameList = new ArrayList<>(); - List columnTypeList = new ArrayList<>(); - Map columnNameIndex = new HashMap<>(); - Statement stmt = connection.createStatement(); - try { - Field[] fields = new Field[8]; - fields[0] = new Field("", "SCOPE", INT32); - fields[1] = new Field("", COLUMN_NAME, "TEXT"); - fields[2] = new Field("", DATA_TYPE, INT32); - fields[3] = new Field("", TYPE_NAME, "TEXT"); - fields[4] = new Field("", COLUMN_SIZE, INT32); - fields[5] = new Field("", BUFFER_LENGTH, INT32); - fields[6] = new Field("", DECIMAL_DIGITS, INT32); - fields[7] = new Field("", "PSEUDO_COLUMN", INT32); - for (int i = 0; i < fields.length; i++) { - columnNameList.add(fields[i].getName()); - columnTypeList.add(fields[i].getSqlType()); - columnNameIndex.put(fields[i].getName(), i); - } - } catch (Exception e) { - LOGGER.error("get version columns error: {}", e.getMessage()); - } finally { - close(null, stmt); - } - return new IoTDBJDBCResultSet( - stmt, - columnNameList, - columnTypeList, - columnNameIndex, - false, - client, - null, - -1, - sessionId, - null, - null, - (long) 60 * 1000, - false, - zoneId); - } - - @Override - public boolean insertsAreDetected(int type) { - return false; - } - - @Override - public boolean isCatalogAtStart() { - return false; - } - - @Override - public boolean isReadOnly() throws SQLException { - try { - return client.getProperties().isReadOnly; - } catch (TException e) { - LOGGER.error("get is readOnly error: {}", e.getMessage()); - } - throw new SQLException("Can not get the read-only mode"); - } - - @Override - public boolean locatorsUpdateCopy() { - return false; - } - - @Override - public boolean nullPlusNonNullIsNull() { - return false; - } - - @Override - public boolean nullsAreSortedAtEnd() { - return false; - } - - @Override - public boolean nullsAreSortedAtStart() { - return false; - } - - @Override - public boolean nullsAreSortedHigh() { - return false; - } - - @Override - public boolean nullsAreSortedLow() { - return false; - } - - @Override - public boolean othersDeletesAreVisible(int type) { - return true; - } - - @Override - public boolean othersInsertsAreVisible(int type) { - return true; - } - - @Override - public boolean othersUpdatesAreVisible(int type) { - return true; - } - - @Override - public boolean ownDeletesAreVisible(int type) { - return true; - } - - @Override - public boolean ownInsertsAreVisible(int type) { - return true; - } - - @Override - public boolean ownUpdatesAreVisible(int type) { - return true; - } - - @Override - public boolean storesLowerCaseIdentifiers() { - return false; - } - - @Override - public boolean storesLowerCaseQuotedIdentifiers() { - return false; - } - - @Override - public boolean storesMixedCaseIdentifiers() { - return true; - } - - @Override - public boolean storesMixedCaseQuotedIdentifiers() { - return true; - } - - @Override - public boolean storesUpperCaseIdentifiers() { - return false; - } - - @Override - public boolean storesUpperCaseQuotedIdentifiers() { - return false; - } - - @Override - public boolean supportsANSI92EntryLevelSQL() { - return false; - } - - @Override - public boolean supportsANSI92FullSQL() { - return false; - } - - @Override - public boolean supportsANSI92IntermediateSQL() { - return false; - } - - @Override - public boolean supportsAlterTableWithAddColumn() { - return true; - } - - @Override - public boolean supportsAlterTableWithDropColumn() { - return true; - } - - @Override - public boolean supportsBatchUpdates() { - return true; - } - - @Override - public boolean supportsCatalogsInDataManipulation() { - return true; - } - - @Override - public boolean supportsCatalogsInIndexDefinitions() { - return true; - } - - @Override - public boolean supportsCatalogsInPrivilegeDefinitions() { - return true; - } - - @Override - public boolean supportsCatalogsInProcedureCalls() { - return true; - } - - @Override - public boolean supportsCatalogsInTableDefinitions() { - return false; - } - - @Override - public boolean supportsColumnAliasing() { - return true; - } - - @Override - public boolean supportsConvert() { - return false; - } - - @Override - public boolean supportsConvert(int fromType, int toType) { - return false; - } - - @Override - public boolean supportsCoreSQLGrammar() { - return false; - } - - @Override - public boolean supportsCorrelatedSubqueries() { - return false; - } - - @Override - public boolean supportsDataDefinitionAndDataManipulationTransactions() { - return false; - } - - @Override - public boolean supportsDataManipulationTransactionsOnly() { - return true; - } - - @Override - public boolean supportsDifferentTableCorrelationNames() { - return false; - } - - @Override - public boolean supportsExpressionsInOrderBy() { - return true; - } - - @Override - public boolean supportsExtendedSQLGrammar() { - return false; - } - - @Override - public boolean supportsFullOuterJoins() { - return true; - } - - @Override - public boolean supportsGetGeneratedKeys() { - return false; - } - - @Override - public boolean supportsGroupBy() { - return true; - } - - @Override - public boolean supportsGroupByBeyondSelect() { - return true; - } - - @Override - public boolean supportsGroupByUnrelated() { - return true; - } - - @Override - public boolean supportsIntegrityEnhancementFacility() { - return false; - } - - @Override - public boolean supportsLikeEscapeClause() { - return false; - } - - @Override - public boolean supportsLimitedOuterJoins() { - return true; - } - - @Override - public boolean supportsMinimumSQLGrammar() { - return false; - } - - @Override - public boolean supportsMixedCaseIdentifiers() { - return true; - } - - @Override - public boolean supportsMixedCaseQuotedIdentifiers() { - return true; - } - - @Override - public boolean supportsMultipleOpenResults() { - return false; - } - - @Override - public boolean supportsMultipleResultSets() { - return false; - } - - @Override - public boolean supportsMultipleTransactions() { - return true; - } - - @Override - public boolean supportsNamedParameters() { - return false; - } - - @Override - public boolean supportsNonNullableColumns() { - return false; - } - - @Override - public boolean supportsOpenCursorsAcrossCommit() { - return false; - } - - @Override - public boolean supportsOpenCursorsAcrossRollback() { - return false; - } - - @Override - public boolean supportsOpenStatementsAcrossCommit() { - return false; - } - - @Override - public boolean supportsOpenStatementsAcrossRollback() { - return false; - } - - @Override - public boolean supportsOrderByUnrelated() { - return true; - } - - @Override - public boolean supportsOuterJoins() { - return true; - } - - @Override - public boolean supportsPositionedDelete() { - return false; - } - - @Override - public boolean supportsPositionedUpdate() { - return false; - } - - @Override - public boolean supportsResultSetConcurrency(int type, int concurrency) { - return false; - } - - @Override - public boolean supportsResultSetHoldability(int holdability) { - return ResultSet.HOLD_CURSORS_OVER_COMMIT == holdability; - } - - @Override - public boolean supportsResultSetType(int type) throws SQLException { - return ResultSet.FETCH_FORWARD == type || ResultSet.TYPE_FORWARD_ONLY == type; - } - - @Override - public boolean supportsSavepoints() { - return false; - } - - @Override - public boolean supportsSchemasInDataManipulation() { - return false; - } - - @Override - public boolean supportsSchemasInIndexDefinitions() { - return false; - } - - @Override - public boolean supportsSchemasInPrivilegeDefinitions() { - return false; - } - - @Override - public boolean supportsSchemasInProcedureCalls() { - return false; - } - - @Override - public boolean supportsSchemasInTableDefinitions() { - return false; - } - - @Override - public boolean supportsSelectForUpdate() { - return false; - } - - @Override - public boolean supportsStatementPooling() { - return false; - } - - @Override - public boolean supportsStoredFunctionsUsingCallSyntax() { - return false; - } - - @Override - public boolean supportsStoredProcedures() { - return false; - } - - @Override - public boolean supportsSubqueriesInComparisons() { - return false; - } - - @Override - public boolean supportsSubqueriesInExists() { - return false; - } - - @Override - public boolean supportsSubqueriesInIns() { - return false; - } - - @Override - public boolean supportsSubqueriesInQuantifieds() { - return false; - } - - @Override - public boolean supportsTableCorrelationNames() { - return false; - } - - @Override - public boolean supportsTransactionIsolationLevel(int level) { - return false; - } - - @Override - public boolean supportsTransactions() { - return false; - } - - @Override - public boolean supportsUnion() { - return false; - } - - @Override - public boolean supportsUnionAll() { - return false; - } - - @Override - public boolean updatesAreDetected(int type) { - return false; - } - - @Override - public boolean usesLocalFilePerTable() { - return false; - } - - @Override - public boolean usesLocalFiles() { - return false; + public String getSchemaTerm() throws SQLException { + return "storage group"; } /** diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java index e5c0e60e84187..dad827a8defec 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java @@ -26,6 +26,7 @@ import org.apache.iotdb.service.rpc.thrift.IClientRPCService; import org.apache.iotdb.service.rpc.thrift.TSTracingInfo; +import org.apache.commons.lang3.ObjectUtils; import org.apache.thrift.TException; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; @@ -34,6 +35,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.sql.rowset.serial.SerialBlob; + import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; @@ -57,16 +60,19 @@ import java.sql.Time; import java.sql.Timestamp; import java.time.ZoneId; -import java.util.BitSet; import java.util.Calendar; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.iotdb.rpc.RpcUtils.convertToTimestamp; public class IoTDBJDBCResultSet implements ResultSet { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBJDBCResultSet.class); - protected Statement statement; + protected IoTDBStatement statement; protected SQLWarning warningChain = null; protected List columnTypeList; protected IoTDBRpcDataSet ioTDBRpcDataSet; @@ -79,60 +85,7 @@ public class IoTDBJDBCResultSet implements ResultSet { @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters public IoTDBJDBCResultSet( - Statement statement, - List columnNameList, - List columnTypeList, - Map columnNameIndex, - boolean ignoreTimeStamp, - IClientRPCService.Iface client, - String sql, - long queryId, - long sessionId, - List dataset, - TSTracingInfo tracingInfo, - long timeout, - String operationType, - List columns, - List sgColumns, - BitSet aliasColumnMap, - boolean moreData, - ZoneId zoneId, - Charset charset) - throws SQLException { - this.ioTDBRpcDataSet = - new IoTDBRpcDataSet( - sql, - columnNameList, - columnTypeList, - columnNameIndex, - ignoreTimeStamp, - moreData, - queryId, - ((IoTDBStatement) statement).getStmtId(), - client, - sessionId, - dataset, - statement.getFetchSize(), - timeout, - sgColumns, - aliasColumnMap, - zoneId, - timeFormat); - this.statement = statement; - this.columnTypeList = columnTypeList; - if (tracingInfo != null) { - ioTDBRpcTracingInfo = new IoTDBTracingInfo(); - ioTDBRpcTracingInfo.setTsTracingInfo(tracingInfo); - } - this.operationType = operationType; - this.columns = columns; - this.sgColumns = sgColumns; - this.charset = charset; - } - - @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters - public IoTDBJDBCResultSet( - Statement statement, + IoTDBStatement statement, List columnNameList, List columnTypeList, Map columnNameIndex, @@ -146,7 +99,9 @@ public IoTDBJDBCResultSet( long timeout, boolean moreData, ZoneId zoneId, - Charset charset) + Charset charset, + boolean tableModel, + List columnIndex2TsBlockColumnIndexList) throws SQLException { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( @@ -157,14 +112,17 @@ public IoTDBJDBCResultSet( ignoreTimeStamp, moreData, queryId, - ((IoTDBStatement) statement).getStmtId(), + statement.getStmtId(), client, sessionId, dataSet, statement.getFetchSize(), timeout, zoneId, - timeFormat); + timeFormat, + statement.getTimeFactor(), + tableModel, + columnIndex2TsBlockColumnIndexList); this.statement = statement; this.columnTypeList = columnTypeList; if (tracingInfo != null) { @@ -207,8 +165,11 @@ public IoTDBJDBCResultSet( statement.getFetchSize(), timeout, zoneId, - timeFormat); - this.statement = statement; + timeFormat, + ((IoTDBStatement) statement).getTimeFactor(), + false, + IntStream.range(0, columnNameList.size()).boxed().collect(Collectors.toList())); + this.statement = (IoTDBStatement) statement; this.columnTypeList = columnTypeList; if (tracingInfo != null) { ioTDBRpcTracingInfo = new IoTDBTracingInfo(); @@ -339,12 +300,28 @@ public InputStream getBinaryStream(String arg0) throws SQLException { @Override public Blob getBlob(int arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); + try { + Binary binary = ioTDBRpcDataSet.getBinary(arg0); + if (ObjectUtils.isNotEmpty(binary)) { + return new SerialBlob(binary.getValues()); + } + return null; + } catch (StatementExecutionException e) { + throw new SQLException(e.getMessage()); + } } @Override public Blob getBlob(String arg0) throws SQLException { - throw new SQLException(Constant.METHOD_NOT_SUPPORTED); + try { + Binary binary = ioTDBRpcDataSet.getBinary(arg0); + if (ObjectUtils.isNotEmpty(binary)) { + return new SerialBlob(binary.getValues()); + } + return null; + } catch (StatementExecutionException e) { + throw new SQLException(e.getMessage()); + } } @Override @@ -468,7 +445,7 @@ public Date getDate(String arg0, Calendar arg1) throws SQLException { @Override public double getDouble(int columnIndex) throws SQLException { try { - if (ioTDBRpcDataSet.columnTypeList.get(columnIndex - 1).equals("FLOAT")) { + if (TSDataType.FLOAT == ioTDBRpcDataSet.getDataType(columnIndex)) { return ioTDBRpcDataSet.getFloat(columnIndex); } return getDouble(ioTDBRpcDataSet.findColumnNameByIndex(columnIndex)); @@ -480,10 +457,7 @@ public double getDouble(int columnIndex) throws SQLException { @Override public double getDouble(String columnName) throws SQLException { try { - if (ioTDBRpcDataSet - .columnTypeList - .get(ioTDBRpcDataSet.columnNameList.indexOf(columnName)) - .equals("FLOAT")) { + if (TSDataType.FLOAT == ioTDBRpcDataSet.getDataType(columnName)) { return ioTDBRpcDataSet.getFloat(columnName); } return ioTDBRpcDataSet.getDouble(columnName); @@ -587,9 +561,9 @@ public ResultSetMetaData getMetaData() { nonAlign, sgColumns, operationTypeColumn, - ioTDBRpcDataSet.columnNameList, - ioTDBRpcDataSet.columnTypeList, - ioTDBRpcDataSet.ignoreTimeStamp); + ioTDBRpcDataSet.getColumnNameList(), + ioTDBRpcDataSet.getColumnTypeList(), + ioTDBRpcDataSet.isIgnoreTimeStamp()); } @Override @@ -722,12 +696,12 @@ public String getString(String columnName) throws SQLException { @Override public Time getTime(int columnIndex) throws SQLException { - return new Time(getLong(columnIndex)); + long time = statement.getMilliSecond(getLong(columnIndex)); + return new Time(time); } @Override public Time getTime(String columnName) throws SQLException { - // TODO: timestamp return getTime(findColumn(columnName)); } @@ -743,7 +717,7 @@ public Time getTime(String arg0, Calendar arg1) throws SQLException { @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { - return new Timestamp(getLong(columnIndex)); + return convertToTimestamp(getLong(columnIndex), statement.getTimeFactor()); } @Override @@ -808,7 +782,7 @@ public boolean isBeforeFirst() throws SQLException { @Override public boolean isClosed() { - return ioTDBRpcDataSet.isClosed; + return ioTDBRpcDataSet.isClosed(); } @Override @@ -1292,20 +1266,20 @@ public void updateTimestamp(String arg0, Timestamp arg1) throws SQLException { @Override public boolean wasNull() { - return ioTDBRpcDataSet.lastReadWasNull; + return ioTDBRpcDataSet.isLastReadWasNull(); } protected String getValueByName(String columnName) throws SQLException { try { - return ioTDBRpcDataSet.getValueByName(columnName); - } catch (StatementExecutionException e) { + return ioTDBRpcDataSet.getString(columnName); + } catch (StatementExecutionException | IllegalArgumentException e) { throw new SQLException(e.getMessage()); } } protected Object getObjectByName(String columnName) throws SQLException { try { - return ioTDBRpcDataSet.getObjectByName(columnName); + return ioTDBRpcDataSet.getObject(columnName); } catch (StatementExecutionException e) { throw new SQLException(e.getMessage()); } @@ -1335,7 +1309,7 @@ public String getStatisticsInfoByName(String name) throws Exception { } public boolean isIgnoreTimeStamp() { - return ioTDBRpcDataSet.ignoreTimeStamp; + return ioTDBRpcDataSet.isIgnoreTimeStamp(); } public String getOperationType() { @@ -1350,11 +1324,7 @@ public List getSgColumns() { return sgColumns; } - public String getColumnTypeByIndex(int columnIndex) { - if (!isIgnoreTimeStamp() && columnIndex == 1) { - return TSDataType.TIMESTAMP.name(); - } - - return ioTDBRpcDataSet.columnTypeList.get(columnIndex - 1); + public TSDataType getColumnTypeByIndex(int columnIndex) { + return ioTDBRpcDataSet.getDataType(columnIndex); } } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java deleted file mode 100644 index c6a592c6781cb..0000000000000 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.jdbc; - -import org.apache.iotdb.rpc.RpcUtils; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.service.rpc.thrift.IClientRPCService; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; -import org.apache.iotdb.service.rpc.thrift.TSQueryNonAlignDataSet; -import org.apache.iotdb.service.rpc.thrift.TSTracingInfo; - -import org.apache.thrift.TException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; - -import java.nio.ByteBuffer; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.apache.iotdb.rpc.IoTDBRpcDataSet.START_INDEX; -import static org.apache.iotdb.rpc.IoTDBRpcDataSet.TIMESTAMP_STR; - -public class IoTDBNonAlignJDBCResultSet extends AbstractIoTDBJDBCResultSet { - - private static final int TIMESTAMP_STR_LENGTH = 4; - private static final String EMPTY_STR = ""; - private String operationType = ""; - private TSQueryNonAlignDataSet tsQueryNonAlignDataSet; - private byte[][] times; // used for disable align - private List sgColumns = null; - - // for disable align clause - @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters - IoTDBNonAlignJDBCResultSet( - Statement statement, - List columnNameList, - List columnTypeList, - Map columnNameIndex, - boolean ignoreTimeStamp, - IClientRPCService.Iface client, - String sql, - long queryId, - long sessionId, - TSQueryNonAlignDataSet dataset, - TSTracingInfo tracingInfo, - long timeout, - String operationType, - List sgColumns, - BitSet aliasColumnMap, - ZoneId zoneId) - throws SQLException { - super( - statement, - columnNameList, - columnTypeList, - columnNameIndex, - ignoreTimeStamp, - client, - sql, - queryId, - sessionId, - timeout, - sgColumns, - aliasColumnMap, - zoneId); - times = new byte[columnNameList.size()][Long.BYTES]; - this.operationType = operationType; - ioTDBRpcDataSet.columnNameList = new ArrayList<>(); - ioTDBRpcDataSet.columnTypeList = new ArrayList<>(); - // deduplicate and map - ioTDBRpcDataSet.columnOrdinalMap = new HashMap<>(); - ioTDBRpcDataSet.columnOrdinalMap.put(TIMESTAMP_STR, 1); - ioTDBRpcDataSet.columnTypeDeduplicatedList = new ArrayList<>(); - ioTDBRpcDataSet.columnTypeDeduplicatedList = new ArrayList<>(columnNameIndex.size()); - for (int i = 0; i < columnNameIndex.size(); i++) { - ioTDBRpcDataSet.columnTypeDeduplicatedList.add(null); - } - List newSgColumns = new ArrayList<>(); - for (int i = 0; i < columnNameList.size(); i++) { - String name = ""; - if (sgColumns != null && !sgColumns.isEmpty()) { - name = sgColumns.get(i) + "." + columnNameList.get(i); - newSgColumns.add(sgColumns.get(i)); - newSgColumns.add(sgColumns.get(i)); - } else { - name = columnNameList.get(i); - newSgColumns.add(""); - newSgColumns.add(""); - } - ioTDBRpcDataSet.columnNameList.add(TIMESTAMP_STR + name); - ioTDBRpcDataSet.columnNameList.add(name); - ioTDBRpcDataSet.columnTypeList.add(String.valueOf(TSDataType.INT64)); - ioTDBRpcDataSet.columnTypeList.add(columnTypeList.get(i)); - if (!ioTDBRpcDataSet.columnOrdinalMap.containsKey(name)) { - int index = columnNameIndex.get(name); - ioTDBRpcDataSet.columnOrdinalMap.put(name, index + START_INDEX); - ioTDBRpcDataSet.columnTypeDeduplicatedList.set( - index, TSDataType.valueOf(columnTypeList.get(i))); - } - } - this.sgColumns = newSgColumns; - this.tsQueryNonAlignDataSet = dataset; - if (tracingInfo != null) { - ioTDBRpcTracingInfo = new IoTDBTracingInfo(); - ioTDBRpcTracingInfo.setTsTracingInfo(tracingInfo); - } - } - - @Override - public long getLong(String columnName) throws SQLException { - checkRecord(); - if (columnName.startsWith(TIMESTAMP_STR)) { - String column = columnName.substring(TIMESTAMP_STR_LENGTH); - int index = ioTDBRpcDataSet.columnOrdinalMap.get(column) - START_INDEX; - if (times[index] != null) { - ioTDBRpcDataSet.lastReadWasNull = false; - return BytesUtils.bytesToLong(times[index]); - } else { - ioTDBRpcDataSet.lastReadWasNull = true; - return 0; - } - } - int index = ioTDBRpcDataSet.columnOrdinalMap.get(columnName) - START_INDEX; - if (ioTDBRpcDataSet.values[index] != null) { - ioTDBRpcDataSet.lastReadWasNull = false; - return BytesUtils.bytesToLong(ioTDBRpcDataSet.values[index]); - } else { - ioTDBRpcDataSet.lastReadWasNull = true; - return 0; - } - } - - @Override - protected boolean fetchResults() throws SQLException { - TSFetchResultsReq req = - new TSFetchResultsReq( - ioTDBRpcDataSet.sessionId, - ioTDBRpcDataSet.sql, - ioTDBRpcDataSet.fetchSize, - ioTDBRpcDataSet.queryId, - false); - req.setTimeout(ioTDBRpcDataSet.timeout); - try { - TSFetchResultsResp resp = ioTDBRpcDataSet.client.fetchResults(req); - - rpcUtilsVerifySuccess(resp); - if (!resp.hasResultSet) { - ioTDBRpcDataSet.emptyResultSet = true; - close(); - } else { - tsQueryNonAlignDataSet = resp.getNonAlignQueryDataSet(); - if (tsQueryNonAlignDataSet == null) { - ioTDBRpcDataSet.emptyResultSet = true; - close(); - return false; - } - } - return resp.hasResultSet; - } catch (TException e) { - throw new SQLException( - "Cannot fetch result from server, because of network connection: {} ", e); - } - } - - private static void rpcUtilsVerifySuccess(TSFetchResultsResp resp) throws IoTDBSQLException { - try { - RpcUtils.verifySuccess(resp.getStatus()); - } catch (StatementExecutionException e) { - throw new IoTDBSQLException(e.getMessage(), resp.getStatus()); - } - } - - @Override - protected boolean hasCachedResults() { - return (tsQueryNonAlignDataSet != null && hasTimesRemaining()); - } - - // check if has times remaining for disable align clause - private boolean hasTimesRemaining() { - for (ByteBuffer time : tsQueryNonAlignDataSet.timeList) { - if (time.hasRemaining()) { - return true; - } - } - return false; - } - - @Override - protected void constructOneRow() { - ioTDBRpcDataSet.lastReadWasNull = false; - for (int i = 0; i < tsQueryNonAlignDataSet.timeList.size(); i++) { - times[i] = null; - ioTDBRpcDataSet.values[i] = null; - if (tsQueryNonAlignDataSet.timeList.get(i).remaining() >= Long.BYTES) { - - times[i] = new byte[Long.BYTES]; - - tsQueryNonAlignDataSet.timeList.get(i).get(times[i]); - ByteBuffer valueBuffer = tsQueryNonAlignDataSet.valueList.get(i); - TSDataType dataType = ioTDBRpcDataSet.columnTypeDeduplicatedList.get(i); - switch (dataType) { - case BOOLEAN: - ioTDBRpcDataSet.values[i] = new byte[1]; - valueBuffer.get(ioTDBRpcDataSet.values[i]); - break; - case INT32: - ioTDBRpcDataSet.values[i] = new byte[Integer.BYTES]; - valueBuffer.get(ioTDBRpcDataSet.values[i]); - break; - case INT64: - ioTDBRpcDataSet.values[i] = new byte[Long.BYTES]; - valueBuffer.get(ioTDBRpcDataSet.values[i]); - break; - case FLOAT: - ioTDBRpcDataSet.values[i] = new byte[Float.BYTES]; - valueBuffer.get(ioTDBRpcDataSet.values[i]); - break; - case DOUBLE: - ioTDBRpcDataSet.values[i] = new byte[Double.BYTES]; - valueBuffer.get(ioTDBRpcDataSet.values[i]); - break; - case TEXT: - int length = valueBuffer.getInt(); - ioTDBRpcDataSet.values[i] = ReadWriteIOUtils.readBytes(valueBuffer, length); - break; - default: - throw new UnSupportedDataTypeException( - String.format( - "Data type %s is not supported.", - ioTDBRpcDataSet.columnTypeDeduplicatedList.get(i))); - } - } else { - ioTDBRpcDataSet.values[i] = EMPTY_STR.getBytes(); - } - } - } - - @Override - protected void checkRecord() throws SQLException { - if (Objects.isNull(tsQueryNonAlignDataSet)) { - throw new SQLException("No record remains"); - } - } - - @Override - protected String getValueByName(String columnName) throws SQLException { - checkRecord(); - if (columnName.startsWith(TIMESTAMP_STR)) { - String column = columnName.substring(TIMESTAMP_STR_LENGTH); - int index = ioTDBRpcDataSet.columnOrdinalMap.get(column) - START_INDEX; - if (times[index] == null || times[index].length == 0) { - return null; - } - return String.valueOf(BytesUtils.bytesToLong(times[index])); - } - int index = ioTDBRpcDataSet.columnOrdinalMap.get(columnName) - START_INDEX; - if (index < 0 - || index >= ioTDBRpcDataSet.values.length - || ioTDBRpcDataSet.values[index] == null - || ioTDBRpcDataSet.values[index].length < 1) { - return null; - } - return ioTDBRpcDataSet.getString( - index, ioTDBRpcDataSet.columnTypeDeduplicatedList.get(index), ioTDBRpcDataSet.values); - } - - @Override - protected Object getObjectByName(String columnName) throws SQLException { - checkRecord(); - if (columnName.startsWith(TIMESTAMP_STR)) { - String column = columnName.substring(TIMESTAMP_STR_LENGTH); - int index = ioTDBRpcDataSet.columnOrdinalMap.get(column) - START_INDEX; - if (times[index] == null || times[index].length == 0) { - return null; - } - return BytesUtils.bytesToLong(times[index]); - } - int index = ioTDBRpcDataSet.columnOrdinalMap.get(columnName) - START_INDEX; - if (index < 0 - || index >= ioTDBRpcDataSet.values.length - || ioTDBRpcDataSet.values[index] == null - || ioTDBRpcDataSet.values[index].length < 1) { - return null; - } - return ioTDBRpcDataSet.getObject( - index, ioTDBRpcDataSet.columnTypeDeduplicatedList.get(index), ioTDBRpcDataSet.values); - } - - public String getOperationType() { - return this.operationType; - } - - public List getSgColumns() { - return sgColumns; - } -} diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java index 72e5c8b081865..389d2867c4194 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java @@ -24,9 +24,11 @@ import org.apache.thrift.TException; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.io.StringReader; @@ -235,7 +237,17 @@ public void setBinaryStream(int parameterIndex, InputStream x) throws SQLExcepti @Override public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(Constant.PARAMETER_SUPPORTED); + byte[] bytes = null; + try { + bytes = ReadWriteIOUtils.readBytes(x, length); + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + this.parameters.put(parameterIndex, "X'" + sb.toString() + "'"); + } catch (IOException e) { + throw new SQLException(Constant.PARAMETER_SUPPORTED); + } } @Override @@ -309,7 +321,8 @@ public void setClob(int parameterIndex, Reader reader, long length) throws SQLEx @Override public void setDate(int parameterIndex, Date x) throws SQLException { - throw new SQLException(Constant.PARAMETER_SUPPORTED); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.parameters.put(parameterIndex, "'" + dateFormat.format(x) + "'"); } @Override @@ -370,7 +383,7 @@ public void setNString(int parameterIndex, String value) throws SQLException { @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { - throw new SQLException(Constant.PARAMETER_NOT_NULL); + this.parameters.put(parameterIndex, "NULL"); } @Override @@ -394,6 +407,10 @@ public void setObject(int parameterIndex, Object x) throws SQLException { setBoolean(parameterIndex, (Boolean) x); } else if (x instanceof Timestamp) { setTimestamp(parameterIndex, (Timestamp) x); + } else if (x instanceof Date) { + setDate(parameterIndex, (Date) x); + } else if (x instanceof Blob) { + setBlob(parameterIndex, (Blob) x); } else if (x instanceof Time) { setTime(parameterIndex, (Time) x); } else { @@ -894,9 +911,14 @@ public void setShort(int parameterIndex, short x) throws SQLException { @Override public void setString(int parameterIndex, String x) { - // if the sql is insert and the value is not a string literal, add double quotes - if (sql.trim().toUpperCase().startsWith("INSERT") && !x.startsWith("\"") && !x.endsWith("'")) { - this.parameters.put(parameterIndex, "\"" + x + "\""); + // if the sql is an insert statement and the value is not a string literal, add single quotes + // The table model only supports single quotes, the tree model sql both single and double quotes + if ("table".equalsIgnoreCase(getSqlDialect()) + || ((sql.trim().toUpperCase().startsWith("INSERT") + && !((x.startsWith("'") && x.endsWith("'")) + || ((x.startsWith("\"") && x.endsWith("\"")) + && "tree".equals(getSqlDialect())))))) { + this.parameters.put(parameterIndex, "'" + x + "'"); } else { this.parameters.put(parameterIndex, x); } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java index eaa080aaefe9d..6831264f5c7d4 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java @@ -30,6 +30,7 @@ import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; +import org.apache.commons.lang3.StringUtils; import org.apache.thrift.TException; import org.apache.tsfile.common.conf.TSFileConfig; @@ -44,6 +45,11 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import static org.apache.iotdb.jdbc.Constant.TABLE; +import static org.apache.iotdb.jdbc.Constant.TREE; public class IoTDBStatement implements Statement { @@ -270,18 +276,10 @@ public boolean execute(String sql) throws SQLException { try { return executeSQL(sql); } catch (TException e) { - if (reConnect()) { - try { - return executeSQL(sql); - } catch (TException e2) { - throw new SQLException(e2); - } - } else { - throw new SQLException( - String.format( - "Fail to reconnect to server when executing %s. please check server status", sql), - e); - } + throw new SQLException( + String.format( + "Fail to reconnect to server when executing %s. please check server status", sql), + e); } } @@ -300,6 +298,54 @@ public boolean execute(String arg0, String[] arg1) throws SQLException { throw new SQLException(NOT_SUPPORT_EXECUTE); } + private interface TFunction { + T run() throws TException; + } + + private T callWithRetryAndReconnect(TFunction rpc, Function statusGetter) + throws SQLException, TException { + TException lastTException = null; + T result = null; + int retryAttempt; + int maxRetryCount = 5; + int retryIntervalInMs = 1000; + for (retryAttempt = 0; retryAttempt <= maxRetryCount; retryAttempt++) { + // 1. try to execute the rpc + try { + result = rpc.run(); + lastTException = null; + } catch (TException e) { + result = null; + lastTException = e; + } + + TSStatus status = null; + if (result != null) { + status = statusGetter.apply(result); + } + // success, return immediately + if (status != null && !(status.isSetNeedRetry() && status.isNeedRetry())) { + return result; + } + + // prepare for the next retry + if (lastTException != null) { + reConnect(); + } + try { + TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + if (result == null && lastTException != null) { + throw lastTException; + } + return result; + } + /** * There are two kinds of sql here: (1) query sql (2) update sql. * @@ -314,35 +360,31 @@ private boolean executeSQL(String sql) throws TException, SQLException { } execReq.setFetchSize(rows); execReq.setTimeout((long) queryTimeout * 1000); - TSExecuteStatementResp execResp = client.executeStatementV2(execReq); + TSExecuteStatementResp execResp = + callWithRetryAndReconnect( + () -> client.executeStatementV2(execReq), TSExecuteStatementResp::getStatus); + + if (execResp.isSetOperationType() && execResp.getOperationType().equals("dropDB")) { + connection.changeDefaultDatabase(null); + } try { RpcUtils.verifySuccess(execResp.getStatus()); } catch (StatementExecutionException e) { throw new IoTDBSQLException(e.getMessage(), execResp.getStatus()); } + if (execResp.isSetDatabase()) { + connection.changeDefaultDatabase(execResp.getDatabase()); + } + + if (execResp.isSetTableModel()) { + connection.mayChangeDefaultSqlDialect(execResp.tableModel ? TABLE : TREE); + } + if (execResp.isSetColumns()) { queryId = execResp.getQueryId(); if (execResp.queryResult == null) { - BitSet aliasColumn = listToBitSet(execResp.getAliasColumns()); - this.resultSet = - new IoTDBNonAlignJDBCResultSet( - this, - execResp.getColumns(), - execResp.getDataTypeList(), - execResp.columnNameIndexMap, - execResp.ignoreTimeStamp, - client, - sql, - queryId, - sessionId, - execResp.nonAlignQueryDataSet, - execResp.tracingInfo, - execReq.timeout, - execResp.operationType, - execResp.getSgColumns(), - aliasColumn, - zoneId); + throw new SQLException("execResp.queryResult should never be null."); } else { this.resultSet = new IoTDBJDBCResultSet( @@ -350,7 +392,7 @@ private boolean executeSQL(String sql) throws TException, SQLException { execResp.getColumns(), execResp.getDataTypeList(), execResp.columnNameIndexMap, - execResp.ignoreTimeStamp, + execResp.isIgnoreTimeStamp(), client, sql, queryId, @@ -360,7 +402,9 @@ private boolean executeSQL(String sql) throws TException, SQLException { execReq.timeout, execResp.moreData, zoneId, - charset); + charset, + execResp.isSetTableModel() && execResp.isTableModel(), + execResp.getColumnIndex2TsBlockColumnIndexList()); } return true; } @@ -374,26 +418,18 @@ public int[] executeBatch() throws SQLException { try { return executeBatchSQL(); } catch (TException e) { - if (reConnect()) { - try { - return executeBatchSQL(); - } catch (TException e2) { - throw new SQLException( - "Fail to execute batch sqls after reconnecting. please check server status", e2); - } - } else { - throw new SQLException( - "Fail to reconnect to server when executing batch sqls. please check server status", e); - } + throw new SQLException( + "Fail to reconnect to server when executing batch sqls. please check server status", e); } finally { clearBatch(); } } - private int[] executeBatchSQL() throws TException, BatchUpdateException { + private int[] executeBatchSQL() throws TException, BatchUpdateException, SQLException { isCancelled = false; TSExecuteBatchStatementReq execReq = new TSExecuteBatchStatementReq(sessionId, batchSQLList); - TSStatus execResp = client.executeBatchStatement(execReq); + TSStatus execResp = + callWithRetryAndReconnect(() -> client.executeBatchStatement(execReq), status -> status); int[] result = new int[batchSQLList.size()]; boolean allSuccess = true; StringBuilder message = new StringBuilder(System.lineSeparator()); @@ -420,6 +456,14 @@ private int[] executeBatchSQL() throws TException, BatchUpdateException { message.append(execResp.getMessage()); } } + if (execResp.isSetSubStatus() && execResp.getSubStatus() != null) { + for (TSStatus status : execResp.getSubStatus()) { + if (status.getCode() == TSStatusCode.USE_OR_DROP_DB.getStatusCode()) { + connection.changeDefaultDatabase(status.getMessage()); + break; + } + } + } if (!allSuccess) { throw new BatchUpdateException(message.toString(), result); } @@ -437,20 +481,9 @@ public ResultSet executeQuery(String sql, long timeoutInMS) throws SQLException try { return executeQuerySQL(sql, timeoutInMS); } catch (TException e) { - if (reConnect()) { - try { - return executeQuerySQL(sql, timeoutInMS); - } catch (TException e2) { - throw new SQLException( - "Fail to executeQuery " + sql + "after reconnecting. please check server status", e2); - } - } else { - throw new SQLException( - "Fail to reconnect to server when execute query " - + sql - + ". please check server status", - e); - } + throw new SQLException( + "Fail to reconnect to server when execute query " + sql + ". please check server status", + e); } } @@ -464,7 +497,9 @@ private ResultSet executeQuerySQL(String sql, long timeoutInMS) throws TExceptio execReq.setFetchSize(rows); execReq.setTimeout(timeoutInMS); execReq.setJdbcQuery(true); - TSExecuteStatementResp execResp = client.executeQueryStatementV2(execReq); + TSExecuteStatementResp execResp = + callWithRetryAndReconnect( + () -> client.executeQueryStatementV2(execReq), TSExecuteStatementResp::getStatus); queryId = execResp.getQueryId(); try { RpcUtils.verifySuccess(execResp.getStatus()); @@ -472,29 +507,8 @@ private ResultSet executeQuerySQL(String sql, long timeoutInMS) throws TExceptio throw new IoTDBSQLException(e.getMessage(), execResp.getStatus()); } - BitSet aliasColumn = null; - if (execResp.getAliasColumns() != null && !execResp.getAliasColumns().isEmpty()) { - aliasColumn = listToBitSet(execResp.getAliasColumns()); - } - if (execResp.queryResult == null) { - this.resultSet = - new IoTDBNonAlignJDBCResultSet( - this, - execResp.getColumns(), - execResp.getDataTypeList(), - execResp.columnNameIndexMap, - execResp.ignoreTimeStamp, - client, - sql, - queryId, - sessionId, - execResp.nonAlignQueryDataSet, - execResp.tracingInfo, - execReq.timeout, - execResp.operationType, - execResp.sgColumns, - aliasColumn, - zoneId); + if (!execResp.isSetQueryResult()) { + throw new SQLException("execResp.queryResult should never be null."); } else { this.resultSet = new IoTDBJDBCResultSet( @@ -502,21 +516,19 @@ private ResultSet executeQuerySQL(String sql, long timeoutInMS) throws TExceptio execResp.getColumns(), execResp.getDataTypeList(), execResp.columnNameIndexMap, - execResp.ignoreTimeStamp, + execResp.isIgnoreTimeStamp(), client, sql, queryId, sessionId, - execResp.queryResult, + execResp.getQueryResult(), execResp.tracingInfo, execReq.timeout, - execResp.operationType, - execResp.columns, - execResp.sgColumns, - aliasColumn, execResp.moreData, zoneId, - charset); + charset, + execResp.isSetTableModel() && execResp.isTableModel(), + execResp.getColumnIndex2TsBlockColumnIndexList()); } return resultSet; } @@ -536,21 +548,9 @@ public int executeUpdate(String sql) throws SQLException { try { return executeUpdateSQL(sql); } catch (TException e) { - if (reConnect()) { - try { - return executeUpdateSQL(sql); - } catch (TException e2) { - throw new SQLException( - "Fail to execute update " + sql + "after reconnecting. please check server status", - e2); - } - } else { - throw new SQLException( - "Fail to reconnect to server when execute update " - + sql - + ". please check server status", - e); - } + throw new SQLException( + "Fail to reconnect to server when execute update " + sql + ". please check server status", + e); } } @@ -569,15 +569,18 @@ public int executeUpdate(String arg0, String[] arg1) throws SQLException { throw new SQLException(NOT_SUPPORT_EXECUTE_UPDATE); } - private int executeUpdateSQL(String sql) throws TException, IoTDBSQLException { - TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionId, sql, stmtId); - TSExecuteStatementResp execResp = client.executeUpdateStatement(execReq); + private int executeUpdateSQL(final String sql) + throws TException, IoTDBSQLException, SQLException { + final TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionId, sql, stmtId); + final TSExecuteStatementResp execResp = + callWithRetryAndReconnect( + () -> client.executeUpdateStatement(execReq), TSExecuteStatementResp::getStatus); if (execResp.isSetQueryId()) { queryId = execResp.getQueryId(); } try { RpcUtils.verifySuccess(execResp.getStatus()); - } catch (StatementExecutionException e) { + } catch (final StatementExecutionException e) { throw new IoTDBSQLException(e.getMessage(), execResp.getStatus()); } return 0; @@ -778,4 +781,24 @@ public long getSessionId() { public long getStmtId() { return stmtId; } + + public long getMilliSecond(long time) { + return RpcUtils.getMilliSecond(time, connection.getTimeFactor()); + } + + public int getNanoSecond(long time) { + return RpcUtils.getNanoSecond(time, connection.getTimeFactor()); + } + + public int getTimeFactor() { + return connection.getTimeFactor(); + } + + public String getSqlDialect() { + if (connection != null && StringUtils.isNotBlank(connection.getSqlDialect())) { + return connection.getSqlDialect().toLowerCase(); + } else { + return "tree"; + } + } } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java index dcda9e3071e21..00e46cc340d15 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java @@ -33,10 +33,10 @@ public class Utils { "squid:S5843", "squid:S5998" }) // Regular expressions should not be too complicated - static final Pattern SUFFIX_URL_PATTERN = Pattern.compile("([0-9]{1,5})(/|\\\\?.*=.*(&.*=.*)*)?"); + static final Pattern SUFFIX_URL_PATTERN = Pattern.compile("(/|\\\\?.*=.*(&.*=.*)*)?"); static final String COLON = ":"; - static final String SLASH = "/"; + static final char SLASH = '/'; static final String PARAMETER_SEPARATOR = "?"; static final String RPC_COMPRESS = "rpc_compress"; @@ -59,29 +59,48 @@ static IoTDBConnectionParams parseUrl(String url, Properties info) throws IoTDBU String subURL = url.substring(Config.IOTDB_URL_PREFIX.length()); int i = subURL.lastIndexOf(COLON); host = subURL.substring(0, i); - suffixURL = subURL.substring(i + 1); + params.setHost(host); + i++; + // parse port + int port = 0; + for (; i < subURL.length() && Character.isDigit(subURL.charAt(i)); i++) { + port = port * 10 + (subURL.charAt(i) - '0'); + } + suffixURL = i < subURL.length() ? subURL.substring(i) : ""; + // legal port + if (port >= 1 && port <= 65535) { + params.setPort(port); + + // parse database + if (i < subURL.length() && subURL.charAt(i) == SLASH) { + int endIndex = subURL.indexOf(PARAMETER_SEPARATOR, i + 1); + String database; + if (endIndex <= i + 1) { + if (i + 1 == subURL.length()) { + database = null; + } else { + database = subURL.substring(i + 1); + } + suffixURL = ""; + } else { + database = subURL.substring(i + 1, endIndex); + suffixURL = subURL.substring(endIndex); + } + params.setDb(database); + } - matcher = SUFFIX_URL_PATTERN.matcher(suffixURL); - if (matcher.matches() && parseUrlParam(subURL, info)) { - isUrlLegal = true; + matcher = SUFFIX_URL_PATTERN.matcher(suffixURL); + if (matcher.matches() && parseUrlParam(subURL, info)) { + isUrlLegal = true; + } } } if (!isUrlLegal) { throw new IoTDBURLException( - "Error url format, url should be jdbc:iotdb://anything:port/ or jdbc:iotdb://anything:port?property1=value1&property2=value2"); + "Error url format, url should be jdbc:iotdb://anything:port/[database] or jdbc:iotdb://anything:port[/database]?property1=value1&property2=value2, current url is " + + url); } - params.setHost(host); - - // parse port - String port = suffixURL; - if (suffixURL.contains(PARAMETER_SEPARATOR)) { - port = suffixURL.split("\\" + PARAMETER_SEPARATOR)[0]; - } else if (suffixURL.contains(SLASH)) { - port = suffixURL.substring(0, suffixURL.length() - 1); - } - params.setPort(Integer.parseInt(port)); - if (info.containsKey(Config.AUTH_USER)) { params.setUsername(info.getProperty(Config.AUTH_USER)); } @@ -117,6 +136,10 @@ static IoTDBConnectionParams parseUrl(String url, Properties info) throws IoTDBU if (info.containsKey(Config.TRUST_STORE_PWD)) { params.setTrustStorePwd(info.getProperty(Config.TRUST_STORE_PWD)); } + if (info.containsKey(Config.SQL_DIALECT)) { + params.setSqlDialect(info.getProperty(Config.SQL_DIALECT)); + } + return params; } @@ -154,6 +177,7 @@ private static boolean parseUrlParam(String subURL, Properties info) { case Config.TRUST_STORE_PWD: case Config.VERSION: case Config.NETWORK_TIMEOUT: + case Config.SQL_DIALECT: info.put(key, value); break; case Config.TIME_ZONE: diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/relational/IoTDBRelationalDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/relational/IoTDBRelationalDatabaseMetadata.java new file mode 100644 index 0000000000000..54236998a5a02 --- /dev/null +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/relational/IoTDBRelationalDatabaseMetadata.java @@ -0,0 +1,655 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.jdbc.relational; + +import org.apache.iotdb.jdbc.Field; +import org.apache.iotdb.jdbc.IoTDBAbstractDatabaseMetadata; +import org.apache.iotdb.jdbc.IoTDBConnection; +import org.apache.iotdb.jdbc.IoTDBJDBCResultSet; +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; + +import org.apache.tsfile.enums.TSDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class IoTDBRelationalDatabaseMetadata extends IoTDBAbstractDatabaseMetadata { + + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBRelationalDatabaseMetadata.class); + + private static final String DATABASE_VERSION = + IoTDBRelationalDatabaseMetadata.class.getPackage().getImplementationVersion() != null + ? IoTDBRelationalDatabaseMetadata.class.getPackage().getImplementationVersion() + : "UNKNOWN"; + + public static final String SHOW_TABLES_ERROR_MSG = "Show tables error: {}"; + + public static final String[] allIotdbTableSQLKeywords = { + "ALTER", + "AND", + "AS", + "BETWEEN", + "BY", + "CASE", + "CAST", + "CONSTRAINT", + "CREATE", + "CROSS", + "CUBE", + "CURRENT_CATALOG", + "CURRENT_DATE", + "CURRENT_ROLE", + "CURRENT_SCHEMA", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "CURRENT_USER", + "DEALLOCATE", + "DELETE", + "DESCRIBE", + "DISTINCT", + "DROP", + "ELSE", + "END", + "ESCAPE", + "EXCEPT", + "EXISTS", + "EXTRACT", + "FALSE", + "FOR", + "FROM", + "FULL", + "GROUP", + "GROUPING", + "HAVING", + "IN", + "INNER", + "INSERT", + "INTERSECT", + "INTO", + "IS", + "JOIN", + "JSON_ARRAY", + "JSON_EXISTS", + "JSON_OBJECT", + "JSON_QUERY", + "JSON_TABLE", + "JSON_VALUE", + "LEFT", + "LIKE", + "LISTAGG", + "LOCALTIME", + "LOCALTIMESTAMP", + "NATURAL", + "NORMALIZE", + "NOT", + "NULL", + "ON", + "OR", + "ORDER", + "OUTER", + "PREPARE", + "RECURSIVE", + "RIGHT", + "ROLLUP", + "SELECT", + "SKIP", + "TABLE", + "THEN", + "TRIM", + "TRUE", + "UESCAPE", + "UNION", + "UNNEST", + "USING", + "VALUES", + "WHEN", + "WHERE", + "WITH", + "FILL" + }; + + private static final String[] allIoTDBTableMathFunctions = { + "ABS", "ACOS", "ASIN", "ATAN", "CEIL", "COS", "COSH", "DEGREES", "E", "EXP", "FLOOR", "LN", + "LOG10", "PI", "RADIANS", "ROUND", "SIGN", "SIN", "SINH", "SQRT", "TAN", "TANH" + }; + + static { + try { + TreeMap myKeywordMap = new TreeMap<>(); + for (String allIotdbSQLKeyword : allIotdbTableSQLKeywords) { + myKeywordMap.put(allIotdbSQLKeyword, null); + } + + HashMap sql92KeywordMap = new HashMap<>(sql92Keywords.length); + for (String sql92Keyword : sql92Keywords) { + sql92KeywordMap.put(sql92Keyword, null); + } + + Iterator it = sql92KeywordMap.keySet().iterator(); + while (it.hasNext()) { + myKeywordMap.remove(it.next()); + } + + StringBuilder keywordBuf = new StringBuilder(); + it = myKeywordMap.keySet().iterator(); + if (it.hasNext()) { + keywordBuf.append(it.next()); + } + while (it.hasNext()) { + keywordBuf.append(","); + keywordBuf.append(it.next()); + } + sqlKeywordsThatArentSQL92 = keywordBuf.toString(); + + } catch (Exception e) { + LOGGER.error("Error when initializing SQL keywords: ", e); + throw new RuntimeException(e); + } + } + + public IoTDBRelationalDatabaseMetadata( + IoTDBConnection connection, IClientRPCService.Iface client, long sessionId, ZoneId zoneId) { + super(connection, client, sessionId, zoneId); + } + + @Override + public String getDriverVersion() throws SQLException { + return DATABASE_VERSION; + } + + @Override + public String getNumericFunctions() throws SQLException { + return String.join(",", allIoTDBTableMathFunctions); + } + + @Override + public String getTimeDateFunctions() throws SQLException { + return "DATE_BIN"; + } + + @Override + public ResultSet getTables( + String catalog, String schemaPattern, String tableNamePattern, String[] types) + throws SQLException { + + Statement stmt = this.connection.createStatement(); + boolean legacyMode = true; + + ResultSet rs; + try { + String sql = + String.format( + "select * from information_schema.tables where database like '%s' escape '\\'", + schemaPattern); + rs = stmt.executeQuery(sql); + } catch (SQLException e) { + LOGGER.error(SHOW_TABLES_ERROR_MSG, e.getMessage()); + + try { + String sql = String.format("show tables details from %s", removeEscape(schemaPattern)); + rs = stmt.executeQuery(sql); + legacyMode = false; + } catch (SQLException e1) { + LOGGER.error(SHOW_TABLES_ERROR_MSG, e.getMessage()); + throw e; + } + } finally { + stmt.close(); + } + + // Setup Fields + Field[] fields = new Field[6]; + fields[0] = new Field("", TABLE_SCHEM, "TEXT"); + fields[1] = new Field("", TABLE_NAME, "TEXT"); + fields[2] = new Field("", TABLE_TYPE, "TEXT"); + fields[3] = new Field("", REMARKS, "TEXT"); + fields[4] = new Field("", COLUMN_SIZE, INT32); + fields[5] = new Field("", DECIMAL_DIGITS, INT32); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + // Extract Values + boolean hasResultSet = false; + while (rs.next()) { + hasResultSet = true; + List valueInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + if (i == 0) { + valueInRow.add(schemaPattern); + } else if (i == 1) { + // valueInRow.add(rs.getString(2)); + valueInRow.add(legacyMode ? rs.getString("table_name") : rs.getString("TableName")); + } else if (i == 2) { + valueInRow.add("TABLE"); + } else if (i == 3) { + // String tgtString = ""; + // String ttl = rs.getString("ttl(ms)"); + // tgtString += "TTL(ms): " + ttl; + String comment = legacyMode ? rs.getString("comment") : rs.getString("Comment"); + if (comment != null && !comment.isEmpty()) { + valueInRow.add(comment); + } else { + valueInRow.add(""); + } + } else if (i == 4) { + valueInRow.add(getTypePrecision(fields[i].getSqlType())); + } else if (i == 5) { + valueInRow.add(getTypeScale(fields[i].getSqlType())); + } else { + valueInRow.add("TABLE"); + } + } + valuesList.add(valueInRow); + } + + // Convert Values to ByteBuffer + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(rs, stmt); + } + + return hasResultSet + ? new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId) + : null; + } + + /** + * some tools pass in parameters that will escape '_', but some syntax does not support escaping, + * such as dataGrip + * + * @param pattern eg. information\_schema + * @return eg. information_schema + */ + public static String removeEscape(String pattern) { + return pattern.replaceAll("\\_", "_"); + } + + @Override + public ResultSet getColumns( + String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + throws SQLException { + + Statement stmt = this.connection.createStatement(); + boolean legacyMode = true; + ResultSet rs; + + // Get Table Metadata + try { + String sql = + String.format( + "select * from information_schema.columns where database like '%s' escape '\\' and table_name like '%s' escape '\\'", + schemaPattern, tableNamePattern); + rs = stmt.executeQuery(sql); + } catch (SQLException e) { + LOGGER.error(SHOW_TABLES_ERROR_MSG, e.getMessage()); + + try { + String sql = + String.format( + "desc %s.%s details", removeEscape(schemaPattern), removeEscape(tableNamePattern)); + rs = stmt.executeQuery(sql); + legacyMode = false; + } catch (SQLException e1) { + LOGGER.error(SHOW_TABLES_ERROR_MSG, e.getMessage()); + throw e; + } + + } finally { + stmt.close(); + } + + // Setup Fields + Field[] fields = new Field[24]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", DATA_TYPE, INT32); + fields[5] = new Field("", TYPE_NAME, "TEXT"); + fields[6] = new Field("", COLUMN_SIZE, INT32); + fields[7] = new Field("", BUFFER_LENGTH, INT32); + fields[8] = new Field("", DECIMAL_DIGITS, INT32); + fields[9] = new Field("", NUM_PREC_RADIX, INT32); + fields[10] = new Field("", NULLABLE, INT32); + fields[11] = new Field("", REMARKS, "TEXT"); + fields[12] = new Field("", COLUMN_DEF, "TEXT"); + fields[13] = new Field("", SQL_DATA_TYPE, INT32); + fields[14] = new Field("", SQL_DATETIME_SUB, INT32); + fields[15] = new Field("", CHAR_OCTET_LENGTH, INT32); + fields[16] = new Field("", ORDINAL_POSITION, INT32); + fields[17] = new Field("", IS_NULLABLE, "TEXT"); + fields[18] = new Field("", SCOPE_CATALOG, "TEXT"); + fields[19] = new Field("", SCOPE_SCHEMA, "TEXT"); + fields[20] = new Field("", SCOPE_TABLE, "TEXT"); + fields[21] = new Field("", SOURCE_DATA_TYPE, "TEXT"); + fields[22] = new Field("", IS_AUTOINCREMENT, "TEXT"); + fields[23] = new Field("", IS_GENERATEDCOLUMN, "TEXT"); + + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + // Extract Metadata + int count = 1; + while (rs.next()) { + String columnName = + legacyMode ? rs.getString("column_name") : rs.getString("ColumnName"); // 3 + String type = legacyMode ? rs.getString("datatype") : rs.getString("DataType"); // 4 + List valueInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + if (i == 0) { + valueInRow.add(""); + } else if (i == 1) { + valueInRow.add(schemaPattern); + } else if (i == 2) { + valueInRow.add(tableNamePattern); + } else if (i == 3) { + valueInRow.add(columnName); + } else if (i == 4) { + valueInRow.add(getSQLType(type)); + } else if (i == 5) { + valueInRow.add(type); + } else if (i == 6) { + valueInRow.add(0); + } else if (i == 7) { + valueInRow.add(65535); + } else if (i == 8) { + valueInRow.add(getTypeScale(fields[i].getSqlType())); + } else if (i == 9) { + valueInRow.add(0); + } else if (i == 10) { + if (!columnName.equals("time")) { + valueInRow.add(ResultSetMetaData.columnNullableUnknown); + } else { + valueInRow.add(ResultSetMetaData.columnNoNulls); + } + } else if (i == 11) { + String comment = legacyMode ? rs.getString("comment") : rs.getString("Comment"); + if (comment != null && !comment.isEmpty()) { + valueInRow.add(comment); + } else { + valueInRow.add(""); + } + } else if (i == 12) { + valueInRow.add(""); + } else if (i == 13) { + valueInRow.add(0); + } else if (i == 14) { + valueInRow.add(0); + } else if (i == 15) { + valueInRow.add(65535); + } else if (i == 16) { + valueInRow.add(count++); + } else if (i == 17) { + if (!columnName.equals("time")) { + valueInRow.add("YES"); + } else { + valueInRow.add("NO"); + } + } else if (i == 18) { + valueInRow.add(""); + } else if (i == 19) { + valueInRow.add(""); + } else if (i == 20) { + valueInRow.add(""); + } else if (i == 21) { + valueInRow.add(0); + } else if (i == 22) { + valueInRow.add(""); + } else if (i == 23) { + valueInRow.add(""); + } else { + valueInRow.add(""); + } + } + valuesList.add(valueInRow); + } + + // Convert Values to ByteBuffer + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error(CONVERT_ERROR_MSG, e.getMessage()); + } finally { + close(rs, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public ResultSet getPrimaryKeys(String catalog, String schemaPattern, String tableNamePattern) + throws SQLException { + + Statement stmt = connection.createStatement(); + boolean legacyMode = true; + ResultSet rs; + + try { + String sql = + String.format( + "select * from information_schema.columns where database like '%s' escape '\\' and table_name like '%s' escape '\\' and (category='TAG' or category='TIME')", + schemaPattern, tableNamePattern); + rs = stmt.executeQuery(sql); + } catch (SQLException e) { + LOGGER.error(SHOW_TABLES_ERROR_MSG, e.getMessage()); + + try { + String sql = + String.format( + "desc %s.%s", removeEscape(schemaPattern), removeEscape(tableNamePattern)); + rs = stmt.executeQuery(sql); + legacyMode = false; + } catch (SQLException e1) { + LOGGER.error(SHOW_TABLES_ERROR_MSG, e.getMessage()); + throw e; + } + + } finally { + stmt.close(); + } + + Field[] fields = new Field[6]; + fields[0] = new Field("", TABLE_CAT, "TEXT"); + fields[1] = new Field("", TABLE_SCHEM, "TEXT"); + fields[2] = new Field("", TABLE_NAME, "TEXT"); + fields[3] = new Field("", COLUMN_NAME, "TEXT"); + fields[4] = new Field("", KEY_SEQ, INT32); + fields[5] = new Field("", PK_NAME, "TEXT"); + List tsDataTypeList = + Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT); + List columnNameList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + Map columnNameIndex = new HashMap<>(); + List> valuesList = new ArrayList<>(); + for (int i = 0; i < fields.length; i++) { + columnNameList.add(fields[i].getName()); + columnTypeList.add(fields[i].getSqlType()); + columnNameIndex.put(fields[i].getName(), i); + } + + int count = 1; + while (rs.next()) { + String columnName = legacyMode ? rs.getString("column_name") : rs.getString("ColumnName"); + String category = legacyMode ? rs.getString("category") : rs.getString("Category"); + if (category.equals("TAG") || category.equals("TIME")) { + List valueInRow = new ArrayList<>(); + for (int i = 0; i < fields.length; ++i) { + if (i == 0) { + valueInRow.add(schemaPattern); + } else if (i == 1) { + valueInRow.add(schemaPattern); + } else if (i == 2) { + valueInRow.add(tableNamePattern); + } else if (i == 3) { + valueInRow.add(columnName); + } else if (i == 4) { + valueInRow.add(count++); + } else { + valueInRow.add(PRIMARY); + } + } + valuesList.add(valueInRow); + } + } + + ByteBuffer tsBlock = null; + try { + tsBlock = convertTsBlock(valuesList, tsDataTypeList); + } catch (IOException e) { + LOGGER.error("Get primary keys error: {}", e.getMessage()); + } finally { + close(null, stmt); + } + + return new IoTDBJDBCResultSet( + stmt, + columnNameList, + columnTypeList, + columnNameIndex, + true, + client, + null, + -1, + sessionId, + Collections.singletonList(tsBlock), + null, + (long) 60 * 1000, + false, + zoneId); + } + + @Override + public boolean supportsSchemasInDataManipulation() throws SQLException { + return true; + } + + @Override + public String getIdentifierQuoteString() throws SQLException { + return "\""; + } + + @Override + public String getSchemaTerm() throws SQLException { + return ""; + } +} diff --git a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java index 03f12a8610862..ab29a14139986 100644 --- a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java +++ b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java @@ -41,6 +41,7 @@ import java.sql.Statement; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -70,6 +71,7 @@ public void before() throws Exception { when(connection.createStatement()) .thenReturn(new IoTDBStatement(connection, client, sessionId, zoneID, 0, 1L)); + when(connection.getTimeFactor()).thenReturn(1_000); databaseMetaData = new IoTDBDatabaseMetadata(connection, client, sessionId, zoneID); when(client.executeStatementV2(any(TSExecuteStatementReq.class))).thenReturn(execStatementResp); when(client.getProperties()).thenReturn(properties); @@ -128,17 +130,25 @@ public void testGetCatalogs() throws SQLException, TException { dataTypeList.add("TEXT"); List columnsList = new ArrayList(); columnsList.add("database"); - Map columnNameIndexMap = new HashMap(); + Map columnNameIndexMap = new HashMap<>(); columnNameIndexMap.put("database", 0); when(client.executeQueryStatementV2(any(TSExecuteStatementReq.class))) .thenReturn(execStatementResp); + when(client.closeOperation(any())).thenReturn(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); when(execStatementResp.getStatus()).thenReturn(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); when(execStatementResp.getQueryId()).thenReturn(queryId); when(execStatementResp.getDataTypeList()).thenReturn(dataTypeList); when(execStatementResp.getColumns()).thenReturn(columnsList); + when(execStatementResp.isSetQueryResult()).thenReturn(true); + when(execStatementResp.getQueryResult()).thenReturn(Collections.emptyList()); + when(execStatementResp.isSetTableModel()).thenReturn(false); + + execStatementResp.moreData = false; + when(execStatementResp.isIgnoreTimeStamp()).thenReturn(true); + when(execStatementResp.getColumnIndex2TsBlockColumnIndexList()).thenReturn(Arrays.asList(0)); execStatementResp.columnNameIndexMap = columnNameIndexMap; ResultSet rs = databaseMetaData.getCatalogs(); - assertEquals(2, rs.findColumn("TYPE_CAT")); + assertEquals(1, rs.findColumn("TYPE_CAT")); } @Test diff --git a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java index 9b5a517484157..3fb60b8760da8 100644 --- a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java +++ b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java @@ -126,6 +126,7 @@ public void before() throws Exception { execResp.queryResult = FakedFirstFetchTsBlockResult(); + when(connection.getTimeFactor()).thenReturn(1000); when(connection.isClosed()).thenReturn(false); when(client.executeStatementV2(any(TSExecuteStatementReq.class))).thenReturn(execResp); when(execResp.getQueryId()).thenReturn(queryId); @@ -172,6 +173,16 @@ public void testQuery() throws Exception { when(execResp.getOperationType()).thenReturn("QUERY"); when(execResp.isSetQueryId()).thenReturn(true); when(execResp.getQueryId()).thenReturn(queryId); + when(execResp.isSetTableModel()).thenReturn(false); + when(execResp.isIgnoreTimeStamp()).thenReturn(false); + List columnIndex2TsBlockColumnIndexList = new ArrayList<>(columns.size()); + columnIndex2TsBlockColumnIndexList.add(0); + columnIndex2TsBlockColumnIndexList.add(1); + columnIndex2TsBlockColumnIndexList.add(2); + columnIndex2TsBlockColumnIndexList.add(0); + + when(execResp.getColumnIndex2TsBlockColumnIndexList()) + .thenReturn(columnIndex2TsBlockColumnIndexList); doReturn("FLOAT") .doReturn("INT64") .doReturn("INT32") diff --git a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java index b7d8203cf93b5..1a523d9459f4b 100644 --- a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java +++ b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java @@ -332,7 +332,7 @@ public void testInsertStatement1() throws Exception { @SuppressWarnings("resource") @Test public void testInsertStatement2() throws Exception { - String sql = "INSERT INTO root.ln.wf01.wt01(time,a,b,c,d,e,f) VALUES(?,?,?,?,?,?,?)"; + String sql = "INSERT INTO root.ln.wf01.wt01(time,a,b,c,d,e,f,g,h) VALUES(?,?,?,?,?,?,?,?,?)"; IoTDBPreparedStatement ps = new IoTDBPreparedStatement(connection, client, sessionId, sql, zoneId); @@ -343,13 +343,15 @@ public void testInsertStatement2() throws Exception { ps.setFloat(5, 123.423f); ps.setDouble(6, -1323.0); ps.setString(7, "\"abc\""); + ps.setString(8, "abc"); + ps.setString(9, "'abc'"); ps.execute(); ArgumentCaptor argument = ArgumentCaptor.forClass(TSExecuteStatementReq.class); verify(client).executeStatementV2(argument.capture()); assertEquals( - "INSERT INTO root.ln.wf01.wt01(time,a,b,c,d,e,f) VALUES(2017-11-01T00:13:00,false,123,123234345,123.423,-1323.0,\"abc\")", + "INSERT INTO root.ln.wf01.wt01(time,a,b,c,d,e,f,g,h) VALUES(2017-11-01T00:13:00,false,123,123234345,123.423,-1323.0,\"abc\",'abc','abc')", argument.getValue().getStatement()); } @@ -395,7 +397,7 @@ public void testInsertStatement4() throws Exception { ArgumentCaptor.forClass(TSExecuteStatementReq.class); verify(client).executeStatementV2(argument.capture()); assertEquals( - "INSERT INTO root.ln.wf01.wt02(time,a,b,c,d,e,f) VALUES(2020-01-01T10:10:10,false,123,123234345,123.423,-1323.0,\"abc\")", + "INSERT INTO root.ln.wf01.wt02(time,a,b,c,d,e,f) VALUES(2020-01-01T10:10:10,false,123,123234345,123.423,-1323.0,'abc')", argument.getValue().getStatement()); } } diff --git a/iotdb-client/pom.xml b/iotdb-client/pom.xml index fe67109c928f0..79a64cc7c066f 100644 --- a/iotdb-client/pom.xml +++ b/iotdb-client/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-client pom diff --git a/iotdb-client/service-rpc/pom.xml b/iotdb-client/service-rpc/pom.xml index 693a97e18556d..a12a86678619b 100644 --- a/iotdb-client/service-rpc/pom.xml +++ b/iotdb-client/service-rpc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT service-rpc IoTDB: Client: Service-RPC @@ -60,12 +60,12 @@ org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index 3e405c95d82d4..a40197fac7988 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -36,52 +36,60 @@ import java.nio.ByteBuffer; import java.sql.Timestamp; +import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; -import java.util.BitSet; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.rpc.RpcUtils.convertToTimestamp; +import static org.apache.iotdb.rpc.RpcUtils.getTimePrecision; public class IoTDBRpcDataSet { - public static final String TIMESTAMP_STR = "Time"; - public static final int START_INDEX = 2; - public String sql; - public boolean isClosed = false; - public IClientRPCService.Iface client; - public List columnNameList; // no deduplication - public List columnTypeList; // no deduplication - public Map + private static final String TIMESTAMP_STR = "Time"; + private static final TsBlockSerde SERDE = new TsBlockSerde(); + + private final String sql; + private boolean isClosed = false; + private IClientRPCService.Iface client; + private final List columnNameList; // no deduplication + private final List columnTypeList; // no deduplication + private final Map columnOrdinalMap; // used because the server returns deduplicated columns - public List columnTypeDeduplicatedList; // deduplicated from columnTypeList - public int fetchSize; - public final long timeout; - public boolean hasCachedRecord = false; - public boolean lastReadWasNull; - - // column size - public int columnSize; - - public long sessionId; - public long queryId; - public long statementId; - public long time; - public boolean ignoreTimeStamp; + private final Map columnName2TsBlockColumnIndexMap; + + // column index -> TsBlock column index + private final List columnIndex2TsBlockColumnIndexList; + + private final List dataTypeForTsBlockColumn; + private int fetchSize; + private final long timeout; + private boolean hasCachedRecord = false; + private boolean lastReadWasNull; + + private final long sessionId; + private final long queryId; + private final long statementId; + private long time; + private final boolean ignoreTimeStamp; // indicates that there is still more data in server side and we can call fetchResult to get more - public boolean moreData; - - public static final TsBlockSerde serde = new TsBlockSerde(); - public List queryResult; - public TsBlock curTsBlock; - public int queryResultSize; // the length of queryResult - public int queryResultIndex; // the index of bytebuffer in queryResult - public int tsBlockSize; // the size of current tsBlock - public int tsBlockIndex; // the row index in current tsBlock + private boolean moreData; + + private List queryResult; + private TsBlock curTsBlock; + private int queryResultSize; // the length of queryResult + private int queryResultIndex; // the index of bytebuffer in queryResult + private int tsBlockSize; // the size of current tsBlock + private int tsBlockIndex; // the row index in current tsBlock private final ZoneId zoneId; private final String timeFormat; + private final int timeFactor; + + private final String timePrecision; + @SuppressWarnings({"squid:S3776", "squid:S107"}) // Suppress high Cognitive Complexity warning public IoTDBRpcDataSet( String sql, @@ -98,9 +106,13 @@ public IoTDBRpcDataSet( int fetchSize, long timeout, ZoneId zoneId, - String timeFormat) { + String timeFormat, + int timeFactor, + boolean tableModel, + List columnIndex2TsBlockColumnIndexList) { this.sessionId = sessionId; this.statementId = statementId; + // only used for tree model, table model this field will always be true this.ignoreTimeStamp = ignoreTimeStamp; this.sql = sql; this.queryId = queryId; @@ -108,166 +120,89 @@ public IoTDBRpcDataSet( this.fetchSize = fetchSize; this.timeout = timeout; this.moreData = moreData; - columnSize = columnNameList.size(); this.columnNameList = new ArrayList<>(); this.columnTypeList = new ArrayList<>(); + this.columnOrdinalMap = new HashMap<>(); + this.columnName2TsBlockColumnIndexMap = new HashMap<>(); + int columnStartIndex = 1; + int resultSetColumnSize = columnNameList.size(); + + // newly generated or updated columnIndex2TsBlockColumnIndexList.size() may not be equal to + // columnNameList.size() + // so we need startIndexForColumnIndex2TsBlockColumnIndexList to adjust the mapping relation + int startIndexForColumnIndex2TsBlockColumnIndexList = 0; + + // for Time Column in tree model which should always be the first column and its index for + // TsBlockColumn is -1 if (!ignoreTimeStamp) { this.columnNameList.add(TIMESTAMP_STR); this.columnTypeList.add(String.valueOf(TSDataType.INT64)); - } - // deduplicate and map - this.columnOrdinalMap = new HashMap<>(); - if (!ignoreTimeStamp) { + this.columnName2TsBlockColumnIndexMap.put(TIMESTAMP_STR, -1); this.columnOrdinalMap.put(TIMESTAMP_STR, 1); + if (columnIndex2TsBlockColumnIndexList != null) { + columnIndex2TsBlockColumnIndexList.add(0, -1); + startIndexForColumnIndex2TsBlockColumnIndexList = 1; + } + columnStartIndex++; + resultSetColumnSize++; } - // deduplicate and map - if (columnNameIndex != null) { - int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count(); - this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize); - for (int i = 0; i < deduplicatedColumnSize; i++) { - columnTypeDeduplicatedList.add(null); + if (columnIndex2TsBlockColumnIndexList == null) { + columnIndex2TsBlockColumnIndexList = new ArrayList<>(resultSetColumnSize); + if (!ignoreTimeStamp) { + startIndexForColumnIndex2TsBlockColumnIndexList = 1; + columnIndex2TsBlockColumnIndexList.add(-1); } - for (int i = 0; i < columnNameList.size(); i++) { - String name = columnNameList.get(i); - this.columnNameList.add(name); - this.columnTypeList.add(columnTypeList.get(i)); - if (!columnOrdinalMap.containsKey(name)) { - int index = columnNameIndex.get(name); - if (!columnOrdinalMap.containsValue(index + START_INDEX)) { - columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i))); - } - columnOrdinalMap.put(name, index + START_INDEX); - } - } - } else { - this.columnTypeDeduplicatedList = new ArrayList<>(); - AtomicInteger index = new AtomicInteger(START_INDEX); - for (int i = 0; i < columnNameList.size(); i++) { - String name = columnNameList.get(i); - this.columnNameList.add(name); - String columnType = columnTypeList.get(i); - this.columnTypeList.add(columnType); - columnOrdinalMap.computeIfAbsent( - name, v -> addColumnTypeListReturnIndex(index, TSDataType.valueOf(columnType))); + for (int i = 0, size = columnNameList.size(); i < size; i++) { + columnIndex2TsBlockColumnIndexList.add(i); } } - this.queryResult = queryResult; - this.queryResultSize = 0; - if (queryResult != null) { - queryResultSize = queryResult.size(); + int tsBlockColumnSize = + columnIndex2TsBlockColumnIndexList.stream().mapToInt(Integer::intValue).max().orElse(0) + 1; + this.dataTypeForTsBlockColumn = new ArrayList<>(tsBlockColumnSize); + for (int i = 0; i < tsBlockColumnSize; i++) { + dataTypeForTsBlockColumn.add(null); } - this.queryResultIndex = 0; - this.tsBlockSize = 0; - this.tsBlockIndex = -1; - this.zoneId = zoneId; - this.timeFormat = timeFormat; - } - - public Integer addColumnTypeListReturnIndex(AtomicInteger index, TSDataType dataType) { - columnTypeDeduplicatedList.add(dataType); - return index.getAndIncrement(); - } - @SuppressWarnings({ - "squid:S3776", - "squid:S107" - }) // ignore Cognitive Complexity of methods should not be too high - public IoTDBRpcDataSet( - String sql, - List columnNameList, - List columnTypeList, - Map columnNameIndex, - boolean ignoreTimeStamp, - boolean moreData, - long queryId, - long statementId, - IClientRPCService.Iface client, - long sessionId, - List queryResult, - int fetchSize, - long timeout, - List sgList, - BitSet aliasColumnMap, - ZoneId zoneId, - String timeFormat) { - this.sessionId = sessionId; - this.statementId = statementId; - this.ignoreTimeStamp = ignoreTimeStamp; - this.sql = sql; - this.queryId = queryId; - this.client = client; - this.fetchSize = fetchSize; - this.timeout = timeout; - this.moreData = moreData; - columnSize = columnNameList.size(); - - this.columnNameList = new ArrayList<>(); - this.columnTypeList = new ArrayList<>(); - if (!ignoreTimeStamp) { - this.columnNameList.add(TIMESTAMP_STR); - this.columnTypeList.add(String.valueOf(TSDataType.INT64)); - } - // deduplicate and map - this.columnOrdinalMap = new HashMap<>(); - if (!ignoreTimeStamp) { - this.columnOrdinalMap.put(TIMESTAMP_STR, 1); - } - - // deduplicate and map - if (columnNameIndex != null) { - int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count(); - this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize); - for (int i = 0; i < deduplicatedColumnSize; i++) { - columnTypeDeduplicatedList.add(null); + for (int i = 0, size = columnNameList.size(); i < size; i++) { + String name = columnNameList.get(i); + this.columnNameList.add(name); + this.columnTypeList.add(columnTypeList.get(i)); + int tsBlockColumnIndex = + columnIndex2TsBlockColumnIndexList.get( + startIndexForColumnIndex2TsBlockColumnIndexList + i); + if (tsBlockColumnIndex != -1) { + TSDataType columnType = TSDataType.valueOf(columnTypeList.get(i)); + dataTypeForTsBlockColumn.set(tsBlockColumnIndex, columnType); } - for (int i = 0; i < columnNameList.size(); i++) { - String name; - if (sgList != null - && !sgList.isEmpty() - && (aliasColumnMap == null || !aliasColumnMap.get(i))) { - name = sgList.get(i) + "." + columnNameList.get(i); - } else { - name = columnNameList.get(i); - } - - this.columnNameList.add(name); - this.columnTypeList.add(columnTypeList.get(i)); - // "Time".equals(name) -> to allow the Time column appear in value columns - if (!columnOrdinalMap.containsKey(name) || "Time".equals(name)) { - int index = columnNameIndex.get(name); - if (!columnOrdinalMap.containsValue(index + START_INDEX)) { - columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i))); - } - columnOrdinalMap.put(name, index + START_INDEX); - } - } - } else { - this.columnTypeDeduplicatedList = new ArrayList<>(); - int index = START_INDEX; - for (int i = 0; i < columnNameList.size(); i++) { - String name = columnNameList.get(i); - this.columnNameList.add(name); - this.columnTypeList.add(columnTypeList.get(i)); - if (!columnOrdinalMap.containsKey(name)) { - columnOrdinalMap.put(name, index++); - columnTypeDeduplicatedList.add(TSDataType.valueOf(columnTypeList.get(i))); - } + if (!columnName2TsBlockColumnIndexMap.containsKey(name)) { + columnOrdinalMap.put(name, i + columnStartIndex); + columnName2TsBlockColumnIndexMap.put(name, tsBlockColumnIndex); } } this.queryResult = queryResult; this.queryResultSize = 0; if (queryResult != null) { - this.queryResultSize = queryResult.size(); + queryResultSize = queryResult.size(); } this.queryResultIndex = 0; this.tsBlockSize = 0; this.tsBlockIndex = -1; this.zoneId = zoneId; this.timeFormat = timeFormat; + this.timeFactor = timeFactor; + this.timePrecision = getTimePrecision(timeFactor); + + if (columnIndex2TsBlockColumnIndexList.size() != this.columnNameList.size()) { + throw new IllegalArgumentException( + String.format( + "Size of columnIndex2TsBlockColumnIndexList %s doesn't equal to size of columnNameList %s.", + columnIndex2TsBlockColumnIndexList.size(), this.columnNameList.size())); + } + this.columnIndex2TsBlockColumnIndexList = columnIndex2TsBlockColumnIndexList; } public void close() throws StatementExecutionException, TException { @@ -320,7 +255,11 @@ public boolean next() throws StatementExecutionException, IoTDBConnectionExcepti } public boolean fetchResults() throws StatementExecutionException, IoTDBConnectionException { + if (isClosed) { + throw new IoTDBConnectionException("This DataSet is already closed"); + } TSFetchResultsReq req = new TSFetchResultsReq(sessionId, sql, fetchSize, queryId, true); + req.setStatementId(statementId); req.setTimeout(timeout); try { TSFetchResultsResp resp = client.fetchResultsV2(req); @@ -363,43 +302,38 @@ public void constructOneTsBlock() { lastReadWasNull = false; ByteBuffer byteBuffer = queryResult.get(queryResultIndex); queryResultIndex++; - curTsBlock = serde.deserialize(byteBuffer); + curTsBlock = SERDE.deserialize(byteBuffer); tsBlockIndex = -1; tsBlockSize = curTsBlock.getPositionCount(); } public boolean isNull(int columnIndex) throws StatementExecutionException { - int index = columnOrdinalMap.get(findColumnNameByIndex(columnIndex)) - START_INDEX; - // time column will never be null - if (index < 0) { - return true; - } - return isNull(index, tsBlockIndex); + return isNull(getTsBlockColumnIndexForColumnIndex(columnIndex), tsBlockIndex); } public boolean isNull(String columnName) { - int index = columnOrdinalMap.get(columnName) - START_INDEX; - // time column will never be null - if (index < 0) { - return true; - } - return isNull(index, tsBlockIndex); + return isNull(getTsBlockColumnIndexForColumnName(columnName), tsBlockIndex); } private boolean isNull(int index, int rowNum) { - return curTsBlock.getColumn(index).isNull(rowNum); + // -1 for time column which will never be null + return index >= 0 && curTsBlock.getColumn(index).isNull(rowNum); } public boolean getBoolean(int columnIndex) throws StatementExecutionException { - return getBoolean(findColumnNameByIndex(columnIndex)); + return getBooleanByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } public boolean getBoolean(String columnName) throws StatementExecutionException { + return getBooleanByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private boolean getBooleanByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (!isNull(index, tsBlockIndex)) { + if (!isNull(tsBlockColumnIndex, tsBlockIndex)) { lastReadWasNull = false; - return curTsBlock.getColumn(index).getBoolean(tsBlockIndex); + return curTsBlock.getColumn(tsBlockColumnIndex).getBoolean(tsBlockIndex); } else { lastReadWasNull = true; return false; @@ -407,15 +341,19 @@ public boolean getBoolean(String columnName) throws StatementExecutionException } public double getDouble(int columnIndex) throws StatementExecutionException { - return getDouble(findColumnNameByIndex(columnIndex)); + return getDoubleByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } public double getDouble(String columnName) throws StatementExecutionException { + return getDoubleByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private double getDoubleByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (!isNull(index, tsBlockIndex)) { + if (!isNull(tsBlockColumnIndex, tsBlockIndex)) { lastReadWasNull = false; - return curTsBlock.getColumn(index).getDouble(tsBlockIndex); + return curTsBlock.getColumn(tsBlockColumnIndex).getDouble(tsBlockIndex); } else { lastReadWasNull = true; return 0; @@ -423,15 +361,19 @@ public double getDouble(String columnName) throws StatementExecutionException { } public float getFloat(int columnIndex) throws StatementExecutionException { - return getFloat(findColumnNameByIndex(columnIndex)); + return getFloatByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } public float getFloat(String columnName) throws StatementExecutionException { + return getFloatByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private float getFloatByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (!isNull(index, tsBlockIndex)) { + if (!isNull(tsBlockColumnIndex, tsBlockIndex)) { lastReadWasNull = false; - return curTsBlock.getColumn(index).getFloat(tsBlockIndex); + return curTsBlock.getColumn(tsBlockColumnIndex).getFloat(tsBlockIndex); } else { lastReadWasNull = true; return 0; @@ -439,19 +381,23 @@ public float getFloat(String columnName) throws StatementExecutionException { } public int getInt(int columnIndex) throws StatementExecutionException { - return getInt(findColumnNameByIndex(columnIndex)); + return getIntByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } public int getInt(String columnName) throws StatementExecutionException { + return getIntByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private int getIntByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (!isNull(index, tsBlockIndex)) { + if (!isNull(tsBlockColumnIndex, tsBlockIndex)) { lastReadWasNull = false; - TSDataType type = curTsBlock.getColumn(index).getDataType(); + TSDataType type = curTsBlock.getColumn(tsBlockColumnIndex).getDataType(); if (type == TSDataType.INT64) { - return (int) curTsBlock.getColumn(index).getLong(tsBlockIndex); + return (int) curTsBlock.getColumn(tsBlockColumnIndex).getLong(tsBlockIndex); } else { - return curTsBlock.getColumn(index).getInt(tsBlockIndex); + return curTsBlock.getColumn(tsBlockColumnIndex).getInt(tsBlockIndex); } } else { lastReadWasNull = true; @@ -460,39 +406,52 @@ public int getInt(String columnName) throws StatementExecutionException { } public long getLong(int columnIndex) throws StatementExecutionException { - return getLong(findColumnNameByIndex(columnIndex)); + return getLongByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } public long getLong(String columnName) throws StatementExecutionException { + int index = getTsBlockColumnIndexForColumnName(columnName); + return getLongByTsBlockColumnIndex(index); + } + + private long getLongByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - if (columnName.equals(TIMESTAMP_STR)) { - return curTsBlock.getTimeByIndex(tsBlockIndex); - } - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (!isNull(index, tsBlockIndex)) { + + // take care of time column + if (tsBlockColumnIndex < 0) { lastReadWasNull = false; - TSDataType type = curTsBlock.getColumn(index).getDataType(); - if (type == TSDataType.INT32) { - return curTsBlock.getColumn(index).getInt(tsBlockIndex); + return curTsBlock.getTimeByIndex(tsBlockIndex); + } else { + if (!isNull(tsBlockColumnIndex, tsBlockIndex)) { + lastReadWasNull = false; + TSDataType type = curTsBlock.getColumn(tsBlockColumnIndex).getDataType(); + if (type == TSDataType.INT32) { + return curTsBlock.getColumn(tsBlockColumnIndex).getInt(tsBlockIndex); + } else { + return curTsBlock.getColumn(tsBlockColumnIndex).getLong(tsBlockIndex); + } } else { - return curTsBlock.getColumn(index).getLong(tsBlockIndex); + lastReadWasNull = true; + return 0; } - } else { - lastReadWasNull = true; - return 0; } } public Binary getBinary(int columIndex) throws StatementExecutionException { - return getBinary(findColumnNameByIndex(columIndex)); + return getBinaryTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columIndex)); } public Binary getBinary(String columnName) throws StatementExecutionException { + return getBinaryTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private Binary getBinaryTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (!isNull(index, tsBlockIndex)) { + if (!isNull(tsBlockColumnIndex, tsBlockIndex)) { lastReadWasNull = false; - return curTsBlock.getColumn(index).getBinary(tsBlockIndex); + return curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex); } else { lastReadWasNull = true; return null; @@ -500,70 +459,91 @@ public Binary getBinary(String columnName) throws StatementExecutionException { } public Object getObject(int columnIndex) throws StatementExecutionException { - return getObject(findColumnNameByIndex(columnIndex)); + return getObjectByTsBlockIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } public Object getObject(String columnName) throws StatementExecutionException { - return getObjectByName(columnName); - } - - public String getString(int columnIndex) throws StatementExecutionException { - return getString(findColumnNameByIndex(columnIndex)); - } - - public String getString(String columnName) throws StatementExecutionException { - return getValueByName(columnName); - } - - public Timestamp getTimestamp(int columnIndex) throws StatementExecutionException { - return new Timestamp(getLong(columnIndex)); - } - - public Timestamp getTimestamp(String columnName) throws StatementExecutionException { - return getTimestamp(findColumn(columnName)); + return getObjectByTsBlockIndex(getTsBlockColumnIndexForColumnName(columnName)); } - public TSDataType getDataType(int columnIndex) throws StatementExecutionException { - return getDataType(findColumnNameByIndex(columnIndex)); + private Object getObjectByTsBlockIndex(int tsBlockColumnIndex) + throws StatementExecutionException { + checkRecord(); + if (isNull(tsBlockColumnIndex, tsBlockIndex)) { + lastReadWasNull = true; + return null; + } + lastReadWasNull = false; + TSDataType tsDataType = getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex); + switch (tsDataType) { + case BOOLEAN: + case INT32: + case INT64: + case FLOAT: + case DOUBLE: + return curTsBlock.getColumn(tsBlockColumnIndex).getObject(tsBlockIndex); + case TIMESTAMP: + long timestamp = + (tsBlockColumnIndex == -1 + ? curTsBlock.getTimeByIndex(tsBlockIndex) + : curTsBlock.getColumn(tsBlockColumnIndex).getLong(tsBlockIndex)); + return convertToTimestamp(timestamp, timeFactor); + case TEXT: + case STRING: + return curTsBlock + .getColumn(tsBlockColumnIndex) + .getBinary(tsBlockIndex) + .getStringValue(TSFileConfig.STRING_CHARSET); + case BLOB: + return BytesUtils.parseBlobByteArrayToString( + curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); + case DATE: + return DateUtils.formatDate(curTsBlock.getColumn(tsBlockColumnIndex).getInt(tsBlockIndex)); + default: + return null; + } } - public TSDataType getDataType(String columnName) throws StatementExecutionException { - if (columnName.equals(TIMESTAMP_STR)) { - return TSDataType.INT64; - } - final int index = columnOrdinalMap.get(columnName) - START_INDEX; - return index < 0 || index >= columnTypeDeduplicatedList.size() - ? null - : columnTypeDeduplicatedList.get(index); + public String getString(int columnIndex) throws StatementExecutionException { + return getStringByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); } - public int findColumn(String columnName) { - return columnOrdinalMap.get(columnName); + public String getString(String columnName) throws StatementExecutionException { + return getStringByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); } - public String getValueByName(String columnName) throws StatementExecutionException { + private String getStringByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { checkRecord(); - if (columnName.equals(TIMESTAMP_STR)) { + // to keep compatibility, tree model should return a long value for time column + if (tsBlockColumnIndex == -1) { return String.valueOf(curTsBlock.getTimeByIndex(tsBlockIndex)); } - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (index < 0 || index >= columnTypeDeduplicatedList.size() || isNull(index, tsBlockIndex)) { + if (isNull(tsBlockColumnIndex, tsBlockIndex)) { lastReadWasNull = true; return null; } lastReadWasNull = false; - return getString(index, columnTypeDeduplicatedList.get(index)); + return getString(tsBlockColumnIndex, getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex)); } - public String getString(int index, TSDataType tsDataType) { + private String getString(int index, TSDataType tsDataType) { switch (tsDataType) { case BOOLEAN: return String.valueOf(curTsBlock.getColumn(index).getBoolean(tsBlockIndex)); case INT32: return String.valueOf(curTsBlock.getColumn(index).getInt(tsBlockIndex)); case INT64: + return String.valueOf( + (index == -1 + ? curTsBlock.getTimeByIndex(tsBlockIndex) + : curTsBlock.getColumn(index).getLong(tsBlockIndex))); case TIMESTAMP: - return String.valueOf(curTsBlock.getColumn(index).getLong(tsBlockIndex)); + long timestamp = + (index == -1 + ? curTsBlock.getTimeByIndex(tsBlockIndex) + : curTsBlock.getColumn(index).getLong(tsBlockIndex)); + return RpcUtils.formatDatetime(timeFormat, timePrecision, timestamp, zoneId); case FLOAT: return String.valueOf(curTsBlock.getColumn(index).getFloat(tsBlockIndex)); case DOUBLE: @@ -584,18 +564,50 @@ public String getString(int index, TSDataType tsDataType) { } } - public Object getObjectByName(String columnName) throws StatementExecutionException { - checkRecord(); - if (columnName.equals(TIMESTAMP_STR)) { - return new Timestamp(curTsBlock.getTimeByIndex(tsBlockIndex)); - } - int index = columnOrdinalMap.get(columnName) - START_INDEX; - if (index < 0 || index >= columnTypeDeduplicatedList.size() || isNull(index, tsBlockIndex)) { - lastReadWasNull = true; - return null; - } - lastReadWasNull = false; - return curTsBlock.getColumn(index).getObject(tsBlockIndex); + public Timestamp getTimestamp(int columnIndex) throws StatementExecutionException { + return getTimestampByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); + } + + public Timestamp getTimestamp(String columnName) throws StatementExecutionException { + return getTimestampByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private Timestamp getTimestampByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { + long timestamp = getLongByTsBlockColumnIndex(tsBlockColumnIndex); + return lastReadWasNull ? null : convertToTimestamp(timestamp, timeFactor); + } + + public LocalDate getDate(int columnIndex) throws StatementExecutionException { + return getDateByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); + } + + public LocalDate getDate(String columnName) throws StatementExecutionException { + return getDateByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + public LocalDate getDateByTsBlockColumnIndex(int tsBlockColumnIndex) + throws StatementExecutionException { + int intValue = getIntByTsBlockColumnIndex(tsBlockColumnIndex); + return lastReadWasNull ? null : DateUtils.parseIntToLocalDate(intValue); + } + + public TSDataType getDataType(int columnIndex) { + return getDataTypeByTsBlockColumnIndex(getTsBlockColumnIndexForColumnIndex(columnIndex)); + } + + public TSDataType getDataType(String columnName) { + return getDataTypeByTsBlockColumnIndex(getTsBlockColumnIndexForColumnName(columnName)); + } + + private TSDataType getDataTypeByTsBlockColumnIndex(int tsBlockColumnIndex) { + return tsBlockColumnIndex < 0 + ? TSDataType.TIMESTAMP + : dataTypeForTsBlockColumn.get(tsBlockColumnIndex); + } + + public int findColumn(String columnName) { + return columnOrdinalMap.get(columnName); } public String findColumnNameByIndex(int columnIndex) throws StatementExecutionException { @@ -609,6 +621,19 @@ public String findColumnNameByIndex(int columnIndex) throws StatementExecutionEx return columnNameList.get(columnIndex - 1); } + // return -1 for time column of tree model + private int getTsBlockColumnIndexForColumnName(String columnName) { + Integer index = columnName2TsBlockColumnIndexMap.get(columnName); + if (index == null) { + throw new IllegalArgumentException("Unknown column name: " + columnName); + } + return index; + } + + private int getTsBlockColumnIndexForColumnIndex(int columnIndex) { + return columnIndex2TsBlockColumnIndexList.get(columnIndex - 1); + } + public void checkRecord() throws StatementExecutionException { if (queryResultIndex > queryResultSize || tsBlockIndex >= tsBlockSize @@ -617,4 +642,52 @@ public void checkRecord() throws StatementExecutionException { throw new StatementExecutionException("No record remains"); } } + + public int getValueColumnStartIndex() { + return ignoreTimeStamp ? 0 : 1; + } + + public int getColumnSize() { + return columnNameList.size(); + } + + public List getColumnTypeList() { + return columnTypeList; + } + + public List getColumnNameList() { + return columnNameList; + } + + public boolean isClosed() { + return isClosed; + } + + public int getFetchSize() { + return fetchSize; + } + + public void setFetchSize(int fetchSize) { + this.fetchSize = fetchSize; + } + + public boolean hasCachedRecord() { + return hasCachedRecord; + } + + public void setHasCachedRecord(boolean hasCachedRecord) { + this.hasCachedRecord = hasCachedRecord; + } + + public boolean isLastReadWasNull() { + return lastReadWasNull; + } + + public long getCurrentRowTime() { + return time; + } + + public boolean isIgnoreTimeStamp() { + return ignoreTimeStamp; + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java index 8da65e8249c03..c7db354b5291e 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import java.io.IOException; +import java.util.List; import java.util.Map; public class RedirectException extends IOException { @@ -29,17 +30,27 @@ public class RedirectException extends IOException { private final TEndPoint endPoint; private final Map deviceEndPointMap; + private final List endPointList; public RedirectException(TEndPoint endPoint) { super("later request in same group will be redirected to " + endPoint.toString()); this.endPoint = endPoint; this.deviceEndPointMap = null; + this.endPointList = null; } public RedirectException(Map deviceEndPointMap) { super("later request in same group will be redirected to " + deviceEndPointMap); this.endPoint = null; this.deviceEndPointMap = deviceEndPointMap; + this.endPointList = null; + } + + public RedirectException(List endPointList) { + super("later request in same group will be redirected to " + endPointList); + this.endPoint = null; + this.deviceEndPointMap = null; + this.endPointList = endPointList; } public TEndPoint getEndPoint() { @@ -49,4 +60,8 @@ public TEndPoint getEndPoint() { public Map getDeviceEndPointMap() { return deviceEndPointMap; } + + public List getEndPointList() { + return endPointList; + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java index e48054abb1974..7541f500e45d0 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java @@ -24,16 +24,19 @@ import org.apache.iotdb.service.rpc.thrift.IClientRPCService; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; +import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.Proxy; +import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -63,6 +66,14 @@ public class RpcUtils { public static final long MIN_SHRINK_INTERVAL = 60_000L; + public static final String TIME_PRECISION = "timestamp_precision"; + + public static final String MILLISECOND = "ms"; + + public static final String MICROSECOND = "us"; + + public static final String NANOSECOND = "ns"; + private RpcUtils() { // util class } @@ -102,6 +113,22 @@ public static void verifySuccessWithRedirection(TSStatus status) if (status.isSetRedirectNode()) { throw new RedirectException(status.getRedirectNode()); } + if (status.isSetSubStatus()) { // the resp of insertRelationalTablet may set subStatus + List statusSubStatus = status.getSubStatus(); + List endPointList = new ArrayList<>(statusSubStatus.size()); + int count = 0; + for (TSStatus subStatus : statusSubStatus) { + if (subStatus.isSetRedirectNode()) { + endPointList.add(subStatus.getRedirectNode()); + count++; + } else { + endPointList.add(null); + } + } + if (!endPointList.isEmpty() && count != 0) { + throw new RedirectException(endPointList); + } + } } public static void verifySuccessWithRedirectionForMultiDevices( @@ -275,7 +302,7 @@ public static String parseLongToDateWithPrecision( DateTimeFormatter formatter, long timestamp, ZoneId zoneid, String timestampPrecision) { long integerOfDate; StringBuilder digits; - if ("ms".equals(timestampPrecision)) { + if (MILLISECOND.equals(timestampPrecision)) { if (timestamp > 0 || timestamp % 1000 == 0) { integerOfDate = timestamp / 1000; digits = new StringBuilder(Long.toString(timestamp % 1000)); @@ -293,7 +320,7 @@ public static String parseLongToDateWithPrecision( } } return formatDatetimeStr(datetime, digits); - } else if ("us".equals(timestampPrecision)) { + } else if (MICROSECOND.equals(timestampPrecision)) { if (timestamp > 0 || timestamp % 1000_000 == 0) { integerOfDate = timestamp / 1000_000; digits = new StringBuilder(Long.toString(timestamp % 1000_000)); @@ -342,4 +369,68 @@ public static TSStatus squashResponseStatusList(List responseStatusLis : new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) .setMessage(failedStatus.toString()); } + + public static boolean isUseDatabase(String sql) { + return sql.length() > 4 && "use ".equalsIgnoreCase(sql.substring(0, 4)); + } + + public static boolean isSetSqlDialect(String sql) { + // check if startWith 'set ' + if (sql.length() <= 15 || !"set ".equalsIgnoreCase(sql.substring(0, 4))) { + return false; + } + + // check if the following content of sql is 'sql_dialect' + sql = sql.substring(4).trim(); + if (sql.length() <= 11) { + return false; + } + return sql.substring(0, 11).equalsIgnoreCase("sql_dialect"); + } + + public static long getMilliSecond(long time, int timeFactor) { + return time / timeFactor * 1_000; + } + + public static int getNanoSecond(long time, int timeFactor) { + return (int) (time % timeFactor * (1_000_000_000 / timeFactor)); + } + + public static Timestamp convertToTimestamp(long time, int timeFactor) { + Timestamp res = new Timestamp(getMilliSecond(time, timeFactor)); + res.setNanos(getNanoSecond(time, timeFactor)); + return res; + } + + public static int getTimeFactor(TSOpenSessionResp openResp) { + if (openResp.isSetConfiguration()) { + String precision = openResp.getConfiguration().get(TIME_PRECISION); + if (precision != null) { + switch (precision) { + case MILLISECOND: + return 1_000; + case MICROSECOND: + return 1_000_000; + case NANOSECOND: + return 1_000_000_000; + default: + throw new IllegalArgumentException("Unknown time precision: " + precision); + } + } + } + return 1_000; + } + + public static String getTimePrecision(int timeFactor) { + switch (timeFactor) { + case 1_000: + return MILLISECOND; + case 1_000_000: + return MICROSECOND; + case 1_000_000_000: + return NANOSECOND; + default: + throw new IllegalArgumentException("Unknown time factor: " + timeFactor); + } + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java index b0c55c21bd063..4c7602c869988 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java @@ -130,9 +130,18 @@ protected void readFrame() throws TTransportException { if (size > thriftMaxFrameSize) { close(); - throw new TTransportException( - TTransportException.CORRUPTED_DATA, - "Frame size (" + size + ") larger than protect max size (" + thriftMaxFrameSize + ")!"); + if (size == 1195725856L || size == 1347375956L) { + // if someone sends HTTP GET/POST to this port, the size will be read as the following + throw new TTransportException( + TTransportException.CORRUPTED_DATA, + "Singular frame size (" + + size + + ") detected, you may be sending HTTP GET/POST requests to the Thrift-RPC port, please confirm that you are using the right port"); + } else { + throw new TTransportException( + TTransportException.CORRUPTED_DATA, + "Frame size (" + size + ") larger than protect max size (" + thriftMaxFrameSize + ")!"); + } } readBuffer.fill(underlying, size); } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 80e31e2469de8..3c61ea03326a0 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -38,6 +38,8 @@ public enum TSStatusCode { START_UP_ERROR(203), SHUT_DOWN_ERROR(204), + UNSUPPORTED_SQL_DIALECT(205), + // General Error UNSUPPORTED_OPERATION(300), EXECUTE_STATEMENT_ERROR(301), @@ -51,6 +53,8 @@ public enum TSStatusCode { // Client, REDIRECTION_RECOMMEND(400), + USE_OR_DROP_DB(401), + // Schema Engine DATABASE_NOT_EXIST(500), DATABASE_ALREADY_EXISTS(501), @@ -80,7 +84,14 @@ public enum TSStatusCode { DATABASE_CONFIG_ERROR(525), SCHEMA_QUOTA_EXCEEDED(526), MEASUREMENT_ALREADY_EXISTS_IN_TEMPLATE(527), - ONLY_LOGICAL_VIEW(528), + TYPE_NOT_FOUND(528), + DATABASE_CONFLICT(529), + DATABASE_MODEL(530), + + TABLE_NOT_EXISTS(550), + TABLE_ALREADY_EXISTS(551), + COLUMN_ALREADY_EXISTS(552), + ONLY_LOGICAL_VIEW(560), // Storage Engine SYSTEM_READ_ONLY(600), @@ -97,6 +108,12 @@ public enum TSStatusCode { DISK_SPACE_INSUFFICIENT(611), OVERSIZE_TTL(612), TTL_CONFIG_ERROR(613), + DATA_TYPE_MISMATCH(614), + COLUMN_CATEGORY_MISMATCH(615), + COLUMN_NOT_EXISTS(616), + MEASUREMENT_NAME_CONFLICT(617), + + WAL_ENTRY_TOO_LARGE(620), // Query Engine SQL_PARSE_ERROR(700), @@ -116,6 +133,18 @@ public enum TSStatusCode { NO_SUCH_QUERY(714), QUERY_WAS_KILLED(715), EXPLAIN_ANALYZE_FETCH_ERROR(716), + TOO_MANY_CONCURRENT_QUERIES_ERROR(717), + + OPERATOR_NOT_FOUND(718), + + QUERY_EXECUTION_MEMORY_NOT_ENOUGH(719), + QUERY_TIMEOUT(720), + PLAN_FAILED_NETWORK_PARTITION(721), + + // Arithmetic + NUMERIC_VALUE_OUT_OF_RANGE(750), + DIVISION_BY_ZERO(751), + DATE_OUT_OF_RANGE(752), // Authentication INIT_AUTH_ERROR(800), @@ -135,8 +164,8 @@ public enum TSStatusCode { AUTH_IO_EXCEPTION(815), ILLEGAL_PRIVILEGE(816), NOT_HAS_PRIVILEGE_GRANTOPT(817), - AUTH_OPERATE_EXCEPTION(818), + OPTIMIZER_TIMEOUT(819), // Partition Error MIGRATE_REGION_ERROR(900), @@ -147,6 +176,9 @@ public enum TSStatusCode { REGION_LEADER_CHANGE_ERROR(905), NO_AVAILABLE_REGION_GROUP(906), LACK_PARTITION_ALLOCATION(907), + RECONSTRUCT_REGION_ERROR(908), + EXTEND_REGION_ERROR(909), + REMOVE_REGION_PEER_ERROR(910), // Cluster Manager ADD_CONFIGNODE_ERROR(1000), @@ -160,6 +192,7 @@ public enum TSStatusCode { TRANSFER_LEADER_ERROR(1008), GET_CLUSTER_ID_ERROR(1009), CAN_NOT_CONNECT_CONFIGNODE(1010), + CAN_NOT_CONNECT_AINODE(1011), // Sync, Load TsFile LOAD_FILE_ERROR(1100), @@ -172,12 +205,17 @@ public enum TSStatusCode { PIPE_ERROR(1107), PIPESERVER_ERROR(1108), VERIFY_METADATA_ERROR(1109), + LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION(1110), + LOAD_IDEMPOTENT_CONFLICT_EXCEPTION(1111), + LOAD_USER_CONFLICT_EXCEPTION(1112), // UDF UDF_LOAD_CLASS_ERROR(1200), UDF_DOWNLOAD_ERROR(1201), CREATE_UDF_ON_DATANODE_ERROR(1202), DROP_UDF_ON_DATANODE_ERROR(1203), + CREATE_UDF_ERROR(1204), + DROP_UDF_ERROR(1205), // Trigger CREATE_TRIGGER_ERROR(1300), @@ -196,7 +234,16 @@ public enum TSStatusCode { CQ_ALREADY_EXIST(1402), CQ_UPDATE_LAST_EXEC_TIME_ERROR(1403), - // code 1500-1599 are used by IoTDB-ML + // AI + CREATE_MODEL_ERROR(1500), + DROP_MODEL_ERROR(1501), + MODEL_EXIST_ERROR(1502), + GET_MODEL_INFO_ERROR(1503), + NO_REGISTERED_AI_NODE_ERROR(1504), + MODEL_NOT_FOUND_ERROR(1505), + REGISTER_AI_NODE_ERROR(1506), + AI_NODE_INTERNAL_ERROR(1510), + REMOVE_AI_NODE_ERROR(1511), // Pipe Plugin CREATE_PIPE_PLUGIN_ERROR(1600), @@ -230,6 +277,7 @@ public enum TSStatusCode { PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION(1809), PIPE_RECEIVER_USER_CONFLICT_EXCEPTION(1810), PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED(1811), + PIPE_TRANSFER_SLICE_OUT_OF_ORDER(1812), // Subscription SUBSCRIPTION_VERSION_ERROR(1900), @@ -243,6 +291,7 @@ public enum TSStatusCode { SUBSCRIPTION_UNSUBSCRIBE_ERROR(1908), SUBSCRIPTION_MISSING_CUSTOMER(1909), SHOW_SUBSCRIPTION_ERROR(1910), + SUBSCRIPTION_PIPE_TIMEOUT_ERROR(1911), // Topic CREATE_TOPIC_ERROR(2000), @@ -267,6 +316,7 @@ public enum TSStatusCode { PIPE_CONSENSUS_TYPE_ERROR(2205), CONSENSUS_GROUP_NOT_EXIST(2206), RATIS_READ_UNAVAILABLE(2207), + PIPE_CONSENSUS_CLOSE_ERROR(2208), ; private final int statusCode; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/annotation/TableModel.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/annotation/TableModel.java new file mode 100644 index 0000000000000..7b1638391ef64 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/annotation/TableModel.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that the method is valid only within the subscription module under the table model + * namespace. Otherwise, the behavior is undefined. + */ +@Target({ElementType.METHOD, ElementType.TYPE, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface TableModel {} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java index 3bcb984732cfc..100de1f112623 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java @@ -68,6 +68,18 @@ public String getConsumerGroupId() { return getString(ConsumerConstant.CONSUMER_GROUP_ID_KEY); } + public String getUsername() { + return getString(ConsumerConstant.USERNAME_KEY); + } + + public String getPassword() { + return getString(ConsumerConstant.PASSWORD_KEY); + } + + public String getSqlDialect() { + return getString(ConsumerConstant.SQL_DIALECT_KEY); + } + public void setConsumerId(final String consumerId) { attributes.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java index 3693e89c3bf5c..dd0c583e40d66 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java @@ -25,6 +25,9 @@ public class ConsumerConstant { /////////////////////////////// common /////////////////////////////// + // TODO: hide from the client + public static final String SQL_DIALECT_KEY = "sql-dialect"; + public static final String HOST_KEY = "host"; public static final String PORT_KEY = "port"; public static final String NODE_URLS_KEY = "node-urls"; @@ -50,6 +53,11 @@ public class ConsumerConstant { public static final String FILE_SAVE_FSYNC_KEY = "file-save-fsync"; public static final boolean FILE_SAVE_FSYNC_DEFAULT_VALUE = false; + public static final String THRIFT_MAX_FRAME_SIZE_KEY = "thrift-max-frame-size"; + + public static final String MAX_POLL_PARALLELISM_KEY = "max-poll-parallelism"; + public static final int MAX_POLL_PARALLELISM_DEFAULT_VALUE = 1; + /////////////////////////////// pull consumer /////////////////////////////// public static final String AUTO_COMMIT_KEY = "auto-commit"; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index fc967835dd24f..f71980d629f10 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -33,9 +33,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.apache.iotdb.rpc.subscription.config.TopicConstant.MODE_LIVE_VALUE; -import static org.apache.iotdb.rpc.subscription.config.TopicConstant.MODE_SNAPSHOT_VALUE; - public class TopicConfig extends PipeParameters { public TopicConfig() { @@ -46,20 +43,36 @@ public TopicConfig(final Map attributes) { super(attributes); } + // TODO: hide from the client + // refer to org.apache.iotdb.commons.pipe.config.constant.SystemConstant + private static final String SQL_DIALECT_KEY = "__system.sql-dialect"; + private static final String SQL_DIALECT_TREE_VALUE = "tree"; + private static final String SQL_DIALECT_TABLE_VALUE = "table"; + private static final Map REALTIME_BATCH_MODE_CONFIG = Collections.singletonMap("realtime.mode", "batch"); private static final Map REALTIME_STREAM_MODE_CONFIG = Collections.singletonMap("realtime.mode", "stream"); + private static final Map SINK_TABLET_FORMAT_CONFIG = + Collections.singletonMap("format", "tablet"); + private static final Map SINK_TS_FILE_FORMAT_CONFIG = + Collections.singletonMap("format", "tsfile"); + private static final Map SINK_HYBRID_FORMAT_CONFIG = + Collections.singletonMap("format", "hybrid"); + private static final Map SNAPSHOT_MODE_CONFIG = - Collections.singletonMap("mode", MODE_SNAPSHOT_VALUE); + Collections.singletonMap("mode", TopicConstant.MODE_SNAPSHOT_VALUE); private static final Map LIVE_MODE_CONFIG = - Collections.singletonMap("mode", MODE_LIVE_VALUE); + Collections.singletonMap("mode", TopicConstant.MODE_LIVE_VALUE); + + private static final Map STRICT_MODE_CONFIG = + Collections.singletonMap("mode.strict", "true"); private static final Set LOOSE_RANGE_KEY_SET; static { - Set set = new HashSet<>(2); + final Set set = new HashSet<>(2); set.add("history.loose-range"); set.add("realtime.loose-range"); LOOSE_RANGE_KEY_SET = Collections.unmodifiableSet(set); @@ -77,7 +90,19 @@ public static TopicConfig deserialize(final ByteBuffer buffer) { /////////////////////////////// utilities /////////////////////////////// - public Map getAttributesWithPathOrPattern() { + public boolean isTableTopic() { + return SQL_DIALECT_TABLE_VALUE.equalsIgnoreCase( + attributes.getOrDefault(SQL_DIALECT_KEY, SQL_DIALECT_TREE_VALUE)); + } + + /////////////////////////////// extractor attributes mapping /////////////////////////////// + + public Map getAttributeWithSqlDialect() { + return Collections.singletonMap( + SQL_DIALECT_KEY, attributes.getOrDefault(SQL_DIALECT_KEY, SQL_DIALECT_TREE_VALUE)); + } + + public Map getAttributesWithSourcePathOrPattern() { if (attributes.containsKey(TopicConstant.PATTERN_KEY)) { return Collections.singletonMap( TopicConstant.PATTERN_KEY, attributes.get(TopicConstant.PATTERN_KEY)); @@ -88,7 +113,19 @@ public Map getAttributesWithPathOrPattern() { attributes.getOrDefault(TopicConstant.PATH_KEY, TopicConstant.PATH_DEFAULT_VALUE)); } - public Map getAttributesWithTimeRange() { + public Map getAttributesWithSourceDatabaseAndTableName() { + final Map attributes = new HashMap<>(); + attributes.put( + TopicConstant.DATABASE_KEY, + this.attributes.getOrDefault( + TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE)); + attributes.put( + TopicConstant.TABLE_KEY, + this.attributes.getOrDefault(TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)); + return attributes; + } + + public Map getAttributesWithSourceTimeRange() { final Map attributesWithTimeRange = new HashMap<>(); // there should be no TopicConstant.NOW_TIME_VALUE here @@ -102,25 +139,47 @@ public Map getAttributesWithTimeRange() { return attributesWithTimeRange; } - public Map getAttributesWithRealtimeMode() { - return REALTIME_STREAM_MODE_CONFIG; + public Map getAttributesWithSourceRealtimeMode() { + return REALTIME_STREAM_MODE_CONFIG; // default to stream (hybrid) } public Map getAttributesWithSourceMode() { - return MODE_SNAPSHOT_VALUE.equalsIgnoreCase( + return TopicConstant.MODE_SNAPSHOT_VALUE.equalsIgnoreCase( attributes.getOrDefault(TopicConstant.MODE_KEY, TopicConstant.MODE_DEFAULT_VALUE)) ? SNAPSHOT_MODE_CONFIG : LIVE_MODE_CONFIG; } - public Map getAttributesWithSourceLooseRange() { - final String looseRangeValue = - attributes.getOrDefault( - TopicConstant.LOOSE_RANGE_KEY, TopicConstant.LOOSE_RANGE_DEFAULT_VALUE); - return LOOSE_RANGE_KEY_SET.stream() - .collect(Collectors.toMap(key -> key, key -> looseRangeValue)); + public Map getAttributesWithSourceLooseRangeOrStrict() { + if (attributes.containsKey(TopicConstant.LOOSE_RANGE_KEY) + && !attributes.containsKey(TopicConstant.STRICT_KEY)) { + // for forwards compatibility + final String looseRangeValue = + attributes.getOrDefault( + TopicConstant.LOOSE_RANGE_KEY, TopicConstant.LOOSE_RANGE_DEFAULT_VALUE); + return LOOSE_RANGE_KEY_SET.stream() + .collect(Collectors.toMap(key -> key, key -> looseRangeValue)); + } else { + // only consider strict + return Collections.singletonMap( + TopicConstant.STRICT_KEY, + attributes.getOrDefault(TopicConstant.STRICT_KEY, TopicConstant.STRICT_DEFAULT_VALUE)); + } + } + + public Map getAttributesWithSourcePrefix() { + final Map attributesWithProcessorPrefix = new HashMap<>(); + attributes.forEach( + (key, value) -> { + if (key.toLowerCase().startsWith("source")) { + attributesWithProcessorPrefix.put(key, value); + } + }); + return attributesWithProcessorPrefix; } + /////////////////////////////// processor attributes mapping /////////////////////////////// + public Map getAttributesWithProcessorPrefix() { final Map attributesWithProcessorPrefix = new HashMap<>(); attributes.forEach( @@ -131,4 +190,26 @@ public Map getAttributesWithProcessorPrefix() { }); return attributesWithProcessorPrefix; } + + /////////////////////////////// connector attributes mapping /////////////////////////////// + + public Map getAttributesWithSinkFormat() { + // refer to + // org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector.parseAndCollectEvent(org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent) + return TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE.equalsIgnoreCase( + attributes.getOrDefault(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_DEFAULT_VALUE)) + ? SINK_TS_FILE_FORMAT_CONFIG + : SINK_TABLET_FORMAT_CONFIG; + } + + public Map getAttributesWithSinkPrefix() { + final Map attributesWithProcessorPrefix = new HashMap<>(); + attributes.forEach( + (key, value) -> { + if (key.toLowerCase().startsWith("sink")) { + attributesWithProcessorPrefix.put(key, value); + } + }); + return attributesWithProcessorPrefix; + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java index d159df2983c5c..4e9df56ec4bc2 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java @@ -26,6 +26,11 @@ public class TopicConstant { public static final String PATTERN_KEY = "pattern"; public static final String PATTERN_DEFAULT_VALUE = "root"; + public static final String DATABASE_KEY = "database"; + public static final String TABLE_KEY = "table"; + public static final String DATABASE_DEFAULT_VALUE = ".*"; + public static final String TABLE_DEFAULT_VALUE = ".*"; + public static final String START_TIME_KEY = "start-time"; public static final String END_TIME_KEY = "end-time"; public static final String NOW_TIME_VALUE = "now"; @@ -46,6 +51,9 @@ public class TopicConstant { public static final String LOOSE_RANGE_ALL_VALUE = "all"; public static final String LOOSE_RANGE_DEFAULT_VALUE = ""; + public static final String STRICT_KEY = "strict"; + public static final String STRICT_DEFAULT_VALUE = "true"; + private TopicConstant() { throw new IllegalStateException("Utility class"); } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPayloadExceedException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPayloadExceedException.java new file mode 100644 index 0000000000000..30c397b4e310f --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPayloadExceedException.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public class SubscriptionPayloadExceedException extends SubscriptionRuntimeCriticalException { + + public SubscriptionPayloadExceedException(final String message) { + super(message); + } + + public SubscriptionPayloadExceedException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionPayloadExceedException + && Objects.equals(getMessage(), ((SubscriptionPayloadExceedException) obj).getMessage()) + && Objects.equals( + getTimeStamp(), ((SubscriptionPayloadExceedException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java new file mode 100644 index 0000000000000..26231e4839695 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public class SubscriptionPipeTimeoutException extends SubscriptionTimeoutException { + + public SubscriptionPipeTimeoutException(final String message) { + super(message); + } + + public SubscriptionPipeTimeoutException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionPipeTimeoutException + && Objects.equals(getMessage(), ((SubscriptionPipeTimeoutException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((SubscriptionPipeTimeoutException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java new file mode 100644 index 0000000000000..67c460ecfb30f --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public class SubscriptionPollTimeoutException extends SubscriptionTimeoutException { + + public SubscriptionPollTimeoutException(final String message) { + super(message); + } + + public SubscriptionPollTimeoutException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionPollTimeoutException + && Objects.equals(getMessage(), ((SubscriptionPollTimeoutException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((SubscriptionPollTimeoutException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionRuntimeCriticalException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionRuntimeCriticalException.java index 72124ba3109dd..0e471dca77799 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionRuntimeCriticalException.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionRuntimeCriticalException.java @@ -21,7 +21,7 @@ import java.util.Objects; -public class SubscriptionRuntimeCriticalException extends SubscriptionException { +public class SubscriptionRuntimeCriticalException extends SubscriptionRuntimeException { public SubscriptionRuntimeCriticalException(final String message) { super(message); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java new file mode 100644 index 0000000000000..f6471a74abaf3 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public abstract class SubscriptionTimeoutException extends SubscriptionRuntimeNonCriticalException { + + public static String KEYWORD = "TimeoutException"; + + public SubscriptionTimeoutException(final String message) { + super(message); + } + + public SubscriptionTimeoutException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionTimeoutException + && Objects.equals(getMessage(), ((SubscriptionTimeoutException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((SubscriptionTimeoutException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/ErrorPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/ErrorPayload.java index 7c2a763f1b956..4cf4d3d96b4f4 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/ErrorPayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/ErrorPayload.java @@ -28,8 +28,14 @@ public class ErrorPayload implements SubscriptionPollPayload { + private static final String OUTDATED_ERROR_MSG = "outdated subscription event"; + public static final ErrorPayload OUTDATED_ERROR_PAYLOAD = + new ErrorPayload(OUTDATED_ERROR_MSG, false); + + /** The error message describing the issue. */ private transient String errorMessage; + /** Indicates whether the error is critical. */ private transient boolean critical; public String getErrorMessage() { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileInitPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileInitPayload.java index 04ebd8a89ab23..d0189b41cf507 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileInitPayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileInitPayload.java @@ -28,6 +28,7 @@ public class FileInitPayload implements SubscriptionPollPayload { + /** The name of the file to be initialized. */ private transient String fileName; public String getFileName() { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FilePiecePayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FilePiecePayload.java index 5c915d83c684e..b0590a23b378a 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FilePiecePayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FilePiecePayload.java @@ -30,10 +30,13 @@ public class FilePiecePayload implements SubscriptionPollPayload { + /** The name of the file. */ private transient String fileName; + /** The field to be filled in the next {@link PollFilePayload} request. */ private transient long nextWritingOffset; + /** The piece of the file content. */ private transient byte[] filePiece; public String getFileName() { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileSealPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileSealPayload.java index bec792c2c9dcf..690ecdc5983e8 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileSealPayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileSealPayload.java @@ -19,6 +19,7 @@ package org.apache.iotdb.rpc.subscription.payload.poll; +import org.apache.thrift.annotation.Nullable; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -28,10 +29,15 @@ public class FileSealPayload implements SubscriptionPollPayload { + /** The name of the file to be sealed. */ private transient String fileName; + /** The length of the file. */ private transient long fileLength; + /** The database name of the file (for table model only). */ + @Nullable private transient String databaseName; + public String getFileName() { return fileName; } @@ -40,23 +46,31 @@ public long getFileLength() { return fileLength; } + public String getDatabaseName() { + return databaseName; + } + public FileSealPayload() {} - public FileSealPayload(final String fileName, final long fileLength) { + public FileSealPayload( + final String fileName, final long fileLength, @Nullable final String databaseName) { this.fileName = fileName; this.fileLength = fileLength; + this.databaseName = databaseName; } @Override public void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(fileName, stream); ReadWriteIOUtils.write(fileLength, stream); + ReadWriteIOUtils.write(databaseName, stream); } @Override public SubscriptionPollPayload deserialize(final ByteBuffer buffer) { this.fileName = ReadWriteIOUtils.readString(buffer); this.fileLength = ReadWriteIOUtils.readLong(buffer); + this.databaseName = ReadWriteIOUtils.readString(buffer); return this; } @@ -70,16 +84,23 @@ public boolean equals(final Object obj) { } final FileSealPayload that = (FileSealPayload) obj; return Objects.equals(this.fileName, that.fileName) - && Objects.equals(this.fileLength, that.fileLength); + && Objects.equals(this.fileLength, that.fileLength) + && Objects.equals(this.databaseName, that.databaseName); } @Override public int hashCode() { - return Objects.hash(fileName, fileLength); + return Objects.hash(fileName, fileLength, databaseName); } @Override public String toString() { - return "FileSealPayload{fileName=" + fileName + ", fileLength=" + fileLength + "}"; + return "FileSealPayload{fileName=" + + fileName + + ", fileLength=" + + fileLength + + ", databaseName=" + + databaseName + + "}"; } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollFilePayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollFilePayload.java index 06128c70237a9..2836ac38abdab 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollFilePayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollFilePayload.java @@ -28,18 +28,14 @@ public class PollFilePayload implements SubscriptionPollPayload { - private transient String topicName; - - private transient String fileName; + /** The commit context associated with the {@link SubscriptionPollResponse}. */ + private transient SubscriptionCommitContext commitContext; + /** The offset from which the file content should be read. */ private transient long writingOffset; - public String getTopicName() { - return topicName; - } - - public String getFileName() { - return fileName; + public SubscriptionCommitContext getCommitContext() { + return commitContext; } public long getWritingOffset() { @@ -48,23 +44,20 @@ public long getWritingOffset() { public PollFilePayload() {} - public PollFilePayload(final String topicName, final String fileName, final long writingOffset) { - this.topicName = topicName; - this.fileName = fileName; + public PollFilePayload(final SubscriptionCommitContext commitContext, final long writingOffset) { + this.commitContext = commitContext; this.writingOffset = writingOffset; } @Override public void serialize(final DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(topicName, stream); - ReadWriteIOUtils.write(fileName, stream); + commitContext.serialize(stream); ReadWriteIOUtils.write(writingOffset, stream); } @Override public SubscriptionPollPayload deserialize(final ByteBuffer buffer) { - topicName = ReadWriteIOUtils.readString(buffer); - fileName = ReadWriteIOUtils.readString(buffer); + commitContext = SubscriptionCommitContext.deserialize(buffer); writingOffset = ReadWriteIOUtils.readLong(buffer); return this; } @@ -80,22 +73,19 @@ public boolean equals(final Object obj) { return false; } final PollFilePayload that = (PollFilePayload) obj; - return Objects.equals(this.topicName, that.topicName) - && Objects.equals(this.fileName, that.fileName) + return Objects.equals(this.commitContext, that.commitContext) && Objects.equals(this.writingOffset, that.writingOffset); } @Override public int hashCode() { - return Objects.hash(topicName, fileName, writingOffset); + return Objects.hash(commitContext, writingOffset); } @Override public String toString() { - return "PollFilePayload{topicName=" - + topicName - + ", fileName=" - + fileName + return "PollFilePayload{commitContext=" + + commitContext + ", writingOffset=" + writingOffset + "}"; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollPayload.java index 77f6747bac8e8..443b899497bd1 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollPayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollPayload.java @@ -30,6 +30,7 @@ public class PollPayload implements SubscriptionPollPayload { + /** The set of topic names that need to be polled. */ private transient Set topicNames = new HashSet<>(); public PollPayload() {} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollTabletsPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollTabletsPayload.java new file mode 100644 index 0000000000000..ea53c93c63675 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollTabletsPayload.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.payload.poll; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class PollTabletsPayload implements SubscriptionPollPayload { + + /** The commit context associated with the {@link SubscriptionPollResponse}. */ + private transient SubscriptionCommitContext commitContext; + + /** The index for the next batch of tablets. */ + private transient int offset; + + public SubscriptionCommitContext getCommitContext() { + return commitContext; + } + + public int getOffset() { + return offset; + } + + public PollTabletsPayload() {} + + public PollTabletsPayload(final SubscriptionCommitContext commitContext, final int offset) { + this.commitContext = commitContext; + this.offset = offset; + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + commitContext.serialize(stream); + ReadWriteIOUtils.write(offset, stream); + } + + @Override + public SubscriptionPollPayload deserialize(final ByteBuffer buffer) { + commitContext = SubscriptionCommitContext.deserialize(buffer); + offset = ReadWriteIOUtils.readInt(buffer); + return this; + } + + /////////////////////////////// Object /////////////////////////////// + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final PollTabletsPayload that = (PollTabletsPayload) obj; + return Objects.equals(this.commitContext, that.commitContext) + && Objects.equals(this.offset, that.offset); + } + + @Override + public int hashCode() { + return Objects.hash(commitContext, offset); + } + + @Override + public String toString() { + return "PollTabletsPayload{commitContext=" + commitContext + ", offset=" + offset + "}"; + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java index fce474fff0d7c..3337887b185f5 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java @@ -36,13 +36,20 @@ public class SubscriptionPollRequest { private final transient SubscriptionPollPayload payload; - private final transient long timeoutMs; // unused now + private final transient long timeoutMs; + + /** The maximum size, in bytes, for the response payload. */ + private final transient long maxBytes; public SubscriptionPollRequest( - final short requestType, final SubscriptionPollPayload payload, final long timeoutMs) { + final short requestType, + final SubscriptionPollPayload payload, + final long timeoutMs, + final long maxBytes) { this.requestType = requestType; this.payload = payload; this.timeoutMs = timeoutMs; + this.maxBytes = maxBytes; } public short getRequestType() { @@ -57,6 +64,10 @@ public long getTimeoutMs() { return timeoutMs; } + public long getMaxBytes() { + return maxBytes; + } + //////////////////////////// serialization //////////////////////////// public static ByteBuffer serialize(final SubscriptionPollRequest request) throws IOException { @@ -71,6 +82,7 @@ private void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(requestType, stream); payload.serialize(stream); ReadWriteIOUtils.write(timeoutMs, stream); + ReadWriteIOUtils.write(maxBytes, stream); } public static SubscriptionPollRequest deserialize(final ByteBuffer buffer) { @@ -84,6 +96,9 @@ public static SubscriptionPollRequest deserialize(final ByteBuffer buffer) { case POLL_FILE: payload = new PollFilePayload().deserialize(buffer); break; + case POLL_TABLETS: + payload = new PollTabletsPayload().deserialize(buffer); + break; default: LOGGER.warn("unexpected request type: {}, payload will be null", requestType); break; @@ -93,7 +108,8 @@ public static SubscriptionPollRequest deserialize(final ByteBuffer buffer) { } final long timeoutMs = ReadWriteIOUtils.readLong(buffer); - return new SubscriptionPollRequest(requestType, payload, timeoutMs); + final long maxBytes = ReadWriteIOUtils.readLong(buffer); + return new SubscriptionPollRequest(requestType, payload, timeoutMs, maxBytes); } /////////////////////////////// object /////////////////////////////// @@ -106,6 +122,8 @@ public String toString() { + payload + ", timeoutMs=" + timeoutMs + + ", maxBytes=" + + maxBytes + "}"; } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestType.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestType.java index c034d4dfbc68c..b225faec9e7dd 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestType.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestType.java @@ -26,6 +26,7 @@ public enum SubscriptionPollRequestType { POLL((short) 0), POLL_FILE((short) 1), + POLL_TABLETS((short) 2), ; private final short type; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java index 01d173d274283..06baa30acee9f 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java @@ -27,6 +27,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; public class SubscriptionPollResponse { @@ -110,16 +112,18 @@ public static SubscriptionPollResponse deserialize(final ByteBuffer buffer) { return new SubscriptionPollResponse(responseType, payload, commitContext); } - /////////////////////////////// object /////////////////////////////// + /////////////////////////////// stringify /////////////////////////////// @Override public String toString() { - return "SubscriptionPollResponse{responseType=" - + SubscriptionPollResponseType.valueOf(responseType).toString() - + ", payload=" - + payload - + ", commitContext=" - + commitContext - + "}"; + return "SubscriptionPollResponse" + coreReportMessage(); + } + + protected Map coreReportMessage() { + final Map result = new HashMap<>(); + result.put("responseType", SubscriptionPollResponseType.valueOf(responseType).toString()); + result.put("payload", payload.toString()); + result.put("commitContext", commitContext.toString()); + return result; } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TabletsPayload.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TabletsPayload.java index 6ece7cc691b3d..19290f7e20e25 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TabletsPayload.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TabletsPayload.java @@ -26,40 +26,88 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; public class TabletsPayload implements SubscriptionPollPayload { - protected transient List tablets = new ArrayList<>(); + /** A batch of tablets. */ + private transient Map> tablets; + + /** + * The field to be filled in the next {@link PollTabletsPayload} request. + * + *
    + *
  • If nextOffset is 1, it indicates that the current payload is the first payload (its + * tablets are empty) and the fetching should continue. + *
  • If nextOffset is negative (or zero), it indicates all tablets have been fetched, and + * -nextOffset represents the total number of tablets. + *
+ */ + private transient int nextOffset; public TabletsPayload() {} - public TabletsPayload(final List tablets) { - this.tablets = new CopyOnWriteArrayList<>(tablets); + public TabletsPayload(final List tablets, final int nextOffset) { + if (tablets.isEmpty()) { + this.tablets = Collections.emptyMap(); + } else { + this.tablets = Collections.singletonMap(null, tablets); + } + this.nextOffset = nextOffset; + } + + public TabletsPayload(final Map> tablets, final int nextOffset) { + this.tablets = tablets; + this.nextOffset = nextOffset; } public List getTablets() { + return tablets.values().stream().flatMap(List::stream).collect(Collectors.toList()); + } + + public Map> getTabletsWithDBInfo() { return tablets; } + public int getNextOffset() { + return nextOffset; + } + @Override public void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(tablets.size(), stream); - for (final Tablet tablet : tablets) { - tablet.serialize(stream); + for (Map.Entry> entry : tablets.entrySet()) { + final String databaseName = entry.getKey(); + final List tabletList = entry.getValue(); + ReadWriteIOUtils.write(databaseName, stream); + ReadWriteIOUtils.write(tabletList.size(), stream); + for (final Tablet tablet : tabletList) { + tablet.serialize(stream); + } } + ReadWriteIOUtils.write(nextOffset, stream); } @Override public SubscriptionPollPayload deserialize(final ByteBuffer buffer) { - final List tablets = new ArrayList<>(); + final Map> tabletsWithDBInfo = new HashMap<>(); final int size = ReadWriteIOUtils.readInt(buffer); for (int i = 0; i < size; ++i) { - tablets.add(Tablet.deserialize(buffer)); + final String databaseName = ReadWriteIOUtils.readString(buffer); + final int tabletsSize = ReadWriteIOUtils.readInt(buffer); + final List tablets = new ArrayList<>(); + for (int j = 0; j < tabletsSize; ++j) { + tablets.add(Tablet.deserialize(buffer)); + } + tabletsWithDBInfo.put(databaseName, tablets); } - this.tablets = new CopyOnWriteArrayList<>(tablets); + this.tablets = tabletsWithDBInfo; + this.nextOffset = ReadWriteIOUtils.readInt(buffer); return this; } @@ -72,16 +120,17 @@ public boolean equals(final Object obj) { return false; } final TabletsPayload that = (TabletsPayload) obj; - return Objects.equals(this.tablets, that.tablets); + return Objects.equals(this.tablets, that.tablets) + && Objects.equals(this.nextOffset, that.nextOffset); } @Override public int hashCode() { - return Objects.hash(tablets); + return Objects.hash(tablets, nextOffset); } @Override public String toString() { - return "TabletsPayload{size of tablets=" + tablets.size() + "}"; + return "TabletsPayload{size of tablets=" + tablets.size() + ", nextOffset=" + nextOffset + "}"; } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeHeartbeatResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeHeartbeatResp.java index 0b0f61b915834..62939f20b4f16 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeHeartbeatResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeHeartbeatResp.java @@ -19,13 +19,36 @@ package org.apache.iotdb.rpc.subscription.payload.response; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; public class PipeSubscribeHeartbeatResp extends TPipeSubscribeResp { + private transient Map topics = new HashMap<>(); // subscribed topics + + private transient Map endPoints = new HashMap<>(); // available endpoints + + public Map getTopics() { + return topics; + } + + public Map getEndPoints() { + return endPoints; + } + /////////////////////////////// Thrift /////////////////////////////// /** @@ -42,11 +65,72 @@ public static PipeSubscribeHeartbeatResp toTPipeSubscribeResp(final TSStatus sta return resp; } + /** + * Serialize the incoming parameters into `PipeSubscribeHeartbeatResp`, called by the subscription + * server. + */ + public static PipeSubscribeHeartbeatResp toTPipeSubscribeResp( + final TSStatus status, + final Map topics, + final Map endPoints) + throws IOException { + final PipeSubscribeHeartbeatResp resp = toTPipeSubscribeResp(status); + + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(topics.size(), outputStream); + for (final Map.Entry entry : topics.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + entry.getValue().serialize(outputStream); + } + ReadWriteIOUtils.write(endPoints.size(), outputStream); + for (final Map.Entry entry : endPoints.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue().getIp(), outputStream); + ReadWriteIOUtils.write(entry.getValue().getPort(), outputStream); + } + resp.body = + Collections.singletonList( + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size())); + } + + return resp; + } + /** Deserialize `TPipeSubscribeResp` to obtain parameters, called by the subscription client. */ public static PipeSubscribeHeartbeatResp fromTPipeSubscribeResp( final TPipeSubscribeResp heartbeatResp) { final PipeSubscribeHeartbeatResp resp = new PipeSubscribeHeartbeatResp(); + if (Objects.nonNull(heartbeatResp.body)) { + for (final ByteBuffer byteBuffer : heartbeatResp.body) { + if (Objects.nonNull(byteBuffer) && byteBuffer.hasRemaining()) { + { + final int size = ReadWriteIOUtils.readInt(byteBuffer); + final Map topics = new HashMap<>(); + for (int i = 0; i < size; i++) { + final String topicName = ReadWriteIOUtils.readString(byteBuffer); + final TopicConfig topicConfig = TopicConfig.deserialize(byteBuffer); + topics.put(topicName, topicConfig); + } + resp.topics = topics; + } + { + final int size = ReadWriteIOUtils.readInt(byteBuffer); + final Map endPoints = new HashMap<>(); + for (int i = 0; i < size; i++) { + final int nodeId = ReadWriteIOUtils.readInt(byteBuffer); + final String ip = ReadWriteIOUtils.readString(byteBuffer); + final int port = ReadWriteIOUtils.readInt(byteBuffer); + endPoints.put(nodeId, new TEndPoint(ip, port)); + } + resp.endPoints = endPoints; + } + break; + } + } + } + resp.status = heartbeatResp.status; resp.version = heartbeatResp.version; resp.type = heartbeatResp.type; @@ -66,7 +150,9 @@ public boolean equals(final Object obj) { return false; } final PipeSubscribeHeartbeatResp that = (PipeSubscribeHeartbeatResp) obj; - return Objects.equals(this.status, that.status) + return Objects.equals(this.topics, that.topics) + && Objects.equals(this.endPoints, that.endPoints) + && Objects.equals(this.status, that.status) && this.version == that.version && this.type == that.type && Objects.equals(this.body, that.body); @@ -74,6 +160,6 @@ public boolean equals(final Object obj) { @Override public int hashCode() { - return Objects.hash(status, version, type, body); + return Objects.hash(topics, endPoints, status, version, type, body); } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribePollResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribePollResp.java index f9fb8a076d822..83c001f7d5a0d 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribePollResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribePollResp.java @@ -46,8 +46,6 @@ public static PipeSubscribePollResp toTPipeSubscribeResp( final TSStatus status, final List byteBuffers) { final PipeSubscribePollResp resp = new PipeSubscribePollResp(); - // resp.events = events; - resp.status = status; resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); resp.type = PipeSubscribeResponseType.ACK.getType(); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java index e12d350bda1a0..ad5ed52df51af 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSubscribeResp.java @@ -62,11 +62,7 @@ public static PipeSubscribeSubscribeResp toTPipeSubscribeResp(final TSStatus sta */ public static PipeSubscribeSubscribeResp toTPipeSubscribeResp( final TSStatus status, final Map topics) throws IOException { - final PipeSubscribeSubscribeResp resp = new PipeSubscribeSubscribeResp(); - - resp.status = status; - resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); - resp.type = PipeSubscribeResponseType.ACK.getType(); + final PipeSubscribeSubscribeResp resp = toTPipeSubscribeResp(status); try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java index a6add300611f4..6856983b37d39 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeUnsubscribeResp.java @@ -62,11 +62,7 @@ public static PipeSubscribeUnsubscribeResp toTPipeSubscribeResp(final TSStatus s */ public static PipeSubscribeUnsubscribeResp toTPipeSubscribeResp( final TSStatus status, final Map topics) throws IOException { - final PipeSubscribeUnsubscribeResp resp = new PipeSubscribeUnsubscribeResp(); - - resp.status = status; - resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion(); - resp.type = PipeSubscribeResponseType.ACK.getType(); + final PipeSubscribeUnsubscribeResp resp = toTPipeSubscribeResp(status); try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { diff --git a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java index 12969e96904f2..a698027607455 100644 --- a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java +++ b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java @@ -64,4 +64,14 @@ public void parseLongToDateWithPrecision() { "1970-01-01T07:59:59.999+08:00", RpcUtils.parseLongToDateWithPrecision(formatter, -1, zoneId, "ms")); } + + @Test + public void testIsSetSqlDialect() { + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect=table")); + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); + Assert.assertFalse(RpcUtils.isSetSqlDialect("setsql_dialect =table")); + Assert.assertFalse(RpcUtils.isSetSqlDialect("set sql_dia")); + } } diff --git a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/TElasticFramedTransportTest.java b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/TElasticFramedTransportTest.java new file mode 100644 index 0000000000000..086dc33825065 --- /dev/null +++ b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/TElasticFramedTransportTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc; + +import org.apache.thrift.transport.TByteBuffer; +import org.apache.thrift.transport.TTransportException; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class TElasticFramedTransportTest { + + @Test + public void testSingularSize() { + + try { + TElasticFramedTransport transport = + new TElasticFramedTransport( + new TByteBuffer( + ByteBuffer.wrap("GET 127.0.0.1 HTTP/1.1".getBytes(StandardCharsets.UTF_8))), + 128 * 1024 * 1024, + 512 * 1024 * 1024, + false); + transport.open(); + transport.read(ByteBuffer.allocate(4096)); + fail("Exception expected"); + } catch (TTransportException e) { + assertEquals( + "Singular frame size (1195725856) detected, you may be sending HTTP GET/POST requests to the Thrift-RPC port, please confirm that you are using the right port", + e.getMessage()); + } + + try { + TElasticFramedTransport transport = + new TElasticFramedTransport( + new TByteBuffer( + ByteBuffer.wrap("POST 127.0.0.1 HTTP/1.1".getBytes(StandardCharsets.UTF_8))), + 128 * 1024 * 1024, + 512 * 1024 * 1024, + false); + transport.open(); + transport.read(ByteBuffer.allocate(4096)); + fail("Exception expected"); + } catch (TTransportException e) { + assertEquals( + "Singular frame size (1347375956) detected, you may be sending HTTP GET/POST requests to the Thrift-RPC port, please confirm that you are using the right port", + e.getMessage()); + } + } +} diff --git a/iotdb-client/session/pom.xml b/iotdb-client/session/pom.xml index 4e514bc3265b8..b5b1e8a3a0942 100644 --- a/iotdb-client/session/pom.xml +++ b/iotdb-client/session/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-session IoTDB: Client: Session @@ -37,17 +37,17 @@ org.apache.iotdb service-rpc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb isession - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.tsfile @@ -62,7 +62,7 @@ org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.slf4j diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/AbstractSessionBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/AbstractSessionBuilder.java new file mode 100644 index 0000000000000..d7bf2726425d2 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/AbstractSessionBuilder.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.util.Version; + +import java.time.ZoneId; +import java.util.List; + +public abstract class AbstractSessionBuilder { + + public String host = SessionConfig.DEFAULT_HOST; + public int rpcPort = SessionConfig.DEFAULT_PORT; + public String username = SessionConfig.DEFAULT_USER; + public String pw = SessionConfig.DEFAULT_PASSWORD; + public int fetchSize = SessionConfig.DEFAULT_FETCH_SIZE; + public ZoneId zoneId = null; + public int thriftDefaultBufferSize = SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY; + public int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; + // this field only take effect in write request, nothing to do with any other type requests, + // like query, load and so on. + // if set to true, it means that we may redirect the write request to its corresponding leader + // if set to false, it means that we will only send write request to first available DataNode(it + // may be changed while current DataNode is not available, for example, we may retry to connect + // to another available DataNode) + // so even if enableRedirection is set to false, we may also send write request to another + // datanode while encountering retriable errors in current DataNode + public boolean enableRedirection = SessionConfig.DEFAULT_REDIRECTION_MODE; + public boolean enableRecordsAutoConvertTablet = SessionConfig.DEFAULT_RECORDS_AUTO_CONVERT_TABLET; + public Version version = SessionConfig.DEFAULT_VERSION; + public long timeOut = SessionConfig.DEFAULT_QUERY_TIME_OUT; + + // set to true, means that we will start a background thread to fetch all available (Status is + // not Removing) datanodes in cluster, and these available nodes will be used in retrying stage + public boolean enableAutoFetch = SessionConfig.DEFAULT_ENABLE_AUTO_FETCH; + + public boolean useSSL = false; + public String trustStore; + public String trustStorePwd; + + // max retry count, if set to 0, means that we won't do any retry + // we can use any available DataNodes(fetched in background thread if enableAutoFetch is true, + // or nodeUrls user specified) to retry, even if enableRedirection is false + public int maxRetryCount = SessionConfig.MAX_RETRY_COUNT; + + public long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; + + public List nodeUrls = null; + + public String sqlDialect = SessionConfig.SQL_DIALECT; + + public String database; +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index a552002fdae5e..8283f2319a150 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -63,6 +63,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; @@ -78,6 +79,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -89,6 +91,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -161,6 +164,9 @@ public class Session implements ISession { @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" protected volatile Map deviceIdToEndpoint; + @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" + protected volatile Map tableModelDeviceIdToEndpoint; + @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" protected volatile Map endPointToSessionConnection; @@ -186,6 +192,11 @@ public class Session implements ISession { protected long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; + protected volatile String sqlDialect = SessionConfig.SQL_DIALECT; + + // may be null + protected volatile String database; + private static final String REDIRECT_TWICE = "redirect twice"; private static final String REDIRECT_TWICE_RETRY = "redirect twice, please try again."; @@ -204,6 +215,9 @@ public class Session implements ISession { "All values are null and this submission is ignored,deviceIds are [{}],times are [{}],measurements are [{}]"; private static final String ALL_INSERT_DATA_IS_NULL = "All inserted data is null."; + protected static final String TABLE = "table"; + protected static final String TREE = "tree"; + public Session(String host, int rpcPort) { this( host, @@ -411,6 +425,7 @@ public Session( if (nodeUrls.isEmpty()) { throw new IllegalArgumentException("nodeUrls shouldn't be empty."); } + Collections.shuffle(nodeUrls); this.nodeUrls = nodeUrls; this.username = username; this.password = password; @@ -422,11 +437,12 @@ public Session( this.version = version; } - public Session(Builder builder) { + public Session(AbstractSessionBuilder builder) { if (builder.nodeUrls != null) { if (builder.nodeUrls.isEmpty()) { throw new IllegalArgumentException("nodeUrls shouldn't be empty."); } + Collections.shuffle(builder.nodeUrls); this.nodeUrls = builder.nodeUrls; this.enableQueryRedirection = true; } else { @@ -448,7 +464,9 @@ public Session(Builder builder) { this.enableAutoFetch = builder.enableAutoFetch; this.maxRetryCount = builder.maxRetryCount; this.retryIntervalInMs = builder.retryIntervalInMs; + this.sqlDialect = builder.sqlDialect; this.queryTimeoutInMs = builder.timeOut; + this.database = builder.database; } @Override @@ -525,6 +543,7 @@ public synchronized void open(boolean enableRPCCompression, int connectionTimeou isClosed = false; if (enableRedirection || enableQueryRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); endPointToSessionConnection = new ConcurrentHashMap<>(); endPointToSessionConnection.put(defaultEndPoint, defaultSessionConnection); } @@ -574,6 +593,33 @@ public synchronized void open( isClosed = false; if (enableRedirection || enableQueryRedirection) { this.deviceIdToEndpoint = deviceIdToEndpoint; + this.tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); + endPointToSessionConnection = new ConcurrentHashMap<>(); + endPointToSessionConnection.put(defaultEndPoint, defaultSessionConnection); + } + } + + @Override + public synchronized void open( + boolean enableRPCCompression, + int connectionTimeoutInMs, + Map deviceIdToEndpoint, + Map tableModelDeviceIdToEndpoint, + INodeSupplier nodesSupplier) + throws IoTDBConnectionException { + if (!isClosed) { + return; + } + + this.availableNodes = nodesSupplier; + this.enableRPCCompression = enableRPCCompression; + this.connectionTimeoutInMs = connectionTimeoutInMs; + defaultSessionConnection = constructSessionConnection(this, defaultEndPoint, zoneId); + defaultSessionConnection.setEnableRedirect(enableQueryRedirection); + isClosed = false; + if (enableRedirection || enableQueryRedirection) { + this.deviceIdToEndpoint = deviceIdToEndpoint; + this.tableModelDeviceIdToEndpoint = tableModelDeviceIdToEndpoint; endPointToSessionConnection = new ConcurrentHashMap<>(); endPointToSessionConnection.put(defaultEndPoint, defaultSessionConnection); } @@ -609,10 +655,17 @@ public SessionConnection constructSessionConnection( Session session, TEndPoint endpoint, ZoneId zoneId) throws IoTDBConnectionException { if (endpoint == null) { return new SessionConnection( - session, zoneId, availableNodes, maxRetryCount, retryIntervalInMs); + session, zoneId, availableNodes, maxRetryCount, retryIntervalInMs, sqlDialect, database); } return new SessionConnection( - session, endpoint, zoneId, availableNodes, maxRetryCount, retryIntervalInMs); + session, + endpoint, + zoneId, + availableNodes, + maxRetryCount, + retryIntervalInMs, + sqlDialect, + database); } @Override @@ -945,7 +998,26 @@ private SessionConnection getQuerySessionConnection() { @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { + String previousDB = database; + String previousDialect = sqlDialect; defaultSessionConnection.executeNonQueryStatement(sql); + if ((!Objects.equals(previousDB, database) || !Objects.equals(previousDialect, sqlDialect)) + && endPointToSessionConnection != null) { + Iterator> iterator = + endPointToSessionConnection.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + SessionConnection sessionConnection = entry.getValue(); + if (sessionConnection != defaultSessionConnection) { + try { + sessionConnection.executeNonQueryStatement(sql); + } catch (Throwable t) { + logger.warn("failed to execute '{}' for {}", sql, entry.getKey()); + iterator.remove(); + } + } + } + } } /** @@ -1255,6 +1327,18 @@ private SessionConnection getSessionConnection(String deviceId) { } } + private SessionConnection getSessionConnection(IDeviceID deviceId) { + TEndPoint endPoint; + if (enableRedirection + && tableModelDeviceIdToEndpoint != null + && (endPoint = tableModelDeviceIdToEndpoint.get(deviceId)) != null + && endPointToSessionConnection.containsKey(endPoint)) { + return endPointToSessionConnection.get(endPoint); + } else { + return defaultSessionConnection; + } + } + @Override public String getTimestampPrecision() throws TException { return defaultSessionConnection.getClient().getProperties().getTimestampPrecision(); @@ -1277,8 +1361,7 @@ public void removeBrokenSessionConnection(SessionConnection sessionConnection) { } } } - - if (deviceIdToEndpoint != null) { + if (deviceIdToEndpoint != null && !deviceIdToEndpoint.isEmpty()) { for (Iterator> it = deviceIdToEndpoint.entrySet().iterator(); it.hasNext(); ) { Entry entry = it.next(); @@ -1287,6 +1370,16 @@ public void removeBrokenSessionConnection(SessionConnection sessionConnection) { } } } + if (tableModelDeviceIdToEndpoint != null && !tableModelDeviceIdToEndpoint.isEmpty()) { + for (Iterator> it = + tableModelDeviceIdToEndpoint.entrySet().iterator(); + it.hasNext(); ) { + Entry entry = it.next(); + if (entry.getValue().equals(endPoint)) { + it.remove(); + } + } + } } } @@ -1296,7 +1389,6 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { if (endpoint.ip.equals("0.0.0.0")) { return; } - AtomicReference exceptionReference = new AtomicReference<>(); if (!deviceIdToEndpoint.containsKey(deviceId) || !deviceIdToEndpoint.get(deviceId).equals(endpoint)) { deviceIdToEndpoint.put(deviceId, endpoint); @@ -1308,7 +1400,6 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { try { return constructSessionConnection(this, endpoint, zoneId); } catch (IoTDBConnectionException ex) { - exceptionReference.set(ex); return null; } }); @@ -1318,6 +1409,32 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { } } + private void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { + if (enableRedirection) { + // no need to redirection + if (endpoint.ip.equals("0.0.0.0")) { + return; + } + if (!tableModelDeviceIdToEndpoint.containsKey(deviceId) + || !tableModelDeviceIdToEndpoint.get(deviceId).equals(endpoint)) { + tableModelDeviceIdToEndpoint.put(deviceId, endpoint); + } + SessionConnection connection = + endPointToSessionConnection.computeIfAbsent( + endpoint, + k -> { + try { + return constructSessionConnection(this, endpoint, zoneId); + } catch (IoTDBConnectionException ex) { + return null; + } + }); + if (connection == null) { + tableModelDeviceIdToEndpoint.remove(deviceId); + } + } + } + private void handleQueryRedirection(TEndPoint endPoint) throws IoTDBConnectionException { if (enableQueryRedirection) { AtomicReference exceptionReference = new AtomicReference<>(); @@ -1548,14 +1665,7 @@ public void insertRecords( } } - /** - * When the value is null,filter this,don't use this measurement. - * - * @param times - * @param measurementsList - * @param valuesList - * @param typesList - */ + /** When the value is null,filter this,don't use this measurement. */ private void filterNullValueAndMeasurement( List deviceIds, List times, @@ -1581,15 +1691,7 @@ private void filterNullValueAndMeasurement( } } - /** - * Filter the null value of list。 - * - * @param deviceId - * @param times - * @param measurementsList - * @param typesList - * @param valuesList - */ + /** Filter the null value of list。 */ private void filterNullValueAndMeasurementOfOneDevice( String deviceId, List times, @@ -1614,14 +1716,7 @@ private void filterNullValueAndMeasurementOfOneDevice( } } - /** - * Filter the null value of list。 - * - * @param times - * @param deviceId - * @param measurementsList - * @param valuesList - */ + /** Filter the null value of list。 */ private void filterNullValueAndMeasurementWithStringTypeOfOneDevice( List times, String deviceId, @@ -1646,10 +1741,6 @@ private void filterNullValueAndMeasurementWithStringTypeOfOneDevice( /** * Filter the null object of list。 * - * @param deviceId - * @param measurementsList - * @param types - * @param valuesList * @return true:all value is null;false:not all null value is null. */ private boolean filterNullValueAndMeasurement( @@ -1679,9 +1770,6 @@ private boolean filterNullValueAndMeasurement( * Filter the null object of list。 * * @param prefixPaths devices path。 - * @param times - * @param measurementsList - * @param valuesList * @return true:all values of valuesList are null;false:Not all values of valuesList are null. */ private void filterNullValueAndMeasurementWithStringType( @@ -1709,8 +1797,6 @@ private void filterNullValueAndMeasurementWithStringType( /** * When the value is null,filter this,don't use this measurement. * - * @param valuesList - * @param measurementsList * @return true:all value is null;false:not all null value is null. */ private boolean filterNullValueAndMeasurementWithStringType( @@ -1819,8 +1905,11 @@ private void insertStringRecordsWithLeaderCache( measurementsList.get(i).toString()); } } - - insertByGroup(recordsGroup, SessionConnection::insertRecords); + if (recordsGroup.size() == 1) { + insertOnce(recordsGroup, SessionConnection::insertRecords); + } else { + insertByGroup(recordsGroup, SessionConnection::insertRecords); + } } private TSInsertStringRecordsReq filterAndGenTSInsertStringRecordsReq( @@ -2457,7 +2546,7 @@ private TSInsertStringRecordsOfOneDeviceReq genTSInsertStringRecordsOfOneDeviceR *

e.g. source: [1,2,3,4,5], index:[1,0,3,2,4], return : [2,1,4,3,5] * * @param source Input list - * @param index retuen order + * @param index return order * @param Input type * @return ordered list */ @@ -2506,7 +2595,11 @@ private void insertRecordsWithLeaderCache( measurementsList.get(i)); } } - insertByGroup(recordsGroup, SessionConnection::insertRecords); + if (recordsGroup.size() == 1) { + insertOnce(recordsGroup, SessionConnection::insertRecords); + } else { + insertByGroup(recordsGroup, SessionConnection::insertRecords); + } } private TSInsertRecordsReq filterAndGenTSInsertRecordsReq( @@ -2608,16 +2701,21 @@ public void insertTablet(Tablet tablet) public void insertTablet(Tablet tablet, boolean sorted) throws IoTDBConnectionException, StatementExecutionException { TSInsertTabletReq request = genTSInsertTabletReq(tablet, sorted, false); + insertTabletInternal(tablet, request); + } + + private void insertTabletInternal(Tablet tablet, TSInsertTabletReq request) + throws IoTDBConnectionException, StatementExecutionException { try { - getSessionConnection(tablet.deviceId).insertTablet(request); + getSessionConnection(tablet.getDeviceId()).insertTablet(request); } catch (RedirectException e) { - handleRedirection(tablet.deviceId, e.getEndPoint()); + handleRedirection(tablet.getDeviceId(), e.getEndPoint()); } catch (IoTDBConnectionException e) { if (enableRedirection && !deviceIdToEndpoint.isEmpty() - && deviceIdToEndpoint.get(tablet.deviceId) != null) { - logger.warn(SESSION_CANNOT_CONNECT, deviceIdToEndpoint.get(tablet.deviceId)); - deviceIdToEndpoint.remove(tablet.deviceId); + && deviceIdToEndpoint.get(tablet.getDeviceId()) != null) { + logger.warn(SESSION_CANNOT_CONNECT, deviceIdToEndpoint.get(tablet.getDeviceId())); + deviceIdToEndpoint.remove(tablet.getDeviceId()); // reconnect with default connection try { @@ -2630,6 +2728,189 @@ public void insertTablet(Tablet tablet, boolean sorted) } } + /** + * insert a relational Tablet. Note: This method is for internal use only, we do not guarantee + * compatibility with subsequent versions. + * + * @param tablet data batch + */ + public void insertRelationalTablet(Tablet tablet) + throws IoTDBConnectionException, StatementExecutionException { + if (tablet.getRowSize() == 0) { + return; + } + if (enableRedirection) { + insertRelationalTabletWithLeaderCache(tablet); + } else { + TSInsertTabletReq request = genTSInsertTabletReq(tablet, false, false); + request.setWriteToTable(true); + request.setColumnCategories( + tablet.getColumnTypes().stream() + .map(t -> (byte) t.ordinal()) + .collect(Collectors.toList())); + try { + defaultSessionConnection.insertTablet(request); + } catch (RedirectException ignored) { + } + } + } + + private void insertRelationalTabletWithLeaderCache(Tablet tablet) + throws IoTDBConnectionException, StatementExecutionException { + Map relationalTabletGroup = new HashMap<>(); + if (tableModelDeviceIdToEndpoint.isEmpty()) { + relationalTabletGroup.put(defaultSessionConnection, tablet); + } else if (SessionUtils.isTabletContainsSingleDevice(tablet)) { + relationalTabletGroup.put(getSessionConnection(tablet.getDeviceID(0)), tablet); + } else { + for (int i = 0; i < tablet.getRowSize(); i++) { + IDeviceID iDeviceID = tablet.getDeviceID(i); + final SessionConnection connection = getSessionConnection(iDeviceID); + int finalI = i; + relationalTabletGroup.compute( + connection, + (k, v) -> { + if (v == null) { + List measurements = new ArrayList<>(tablet.getSchemas().size()); + List dataTypes = new ArrayList<>(); + tablet + .getSchemas() + .forEach( + schema -> { + measurements.add(schema.getMeasurementName()); + dataTypes.add(schema.getType()); + }); + v = + new Tablet( + tablet.getTableName(), + measurements, + dataTypes, + tablet.getColumnTypes(), + tablet.getRowSize()); + } + for (int j = 0; j < v.getSchemas().size(); j++) { + v.addValue( + v.getSchemas().get(j).getMeasurementName(), + v.getRowSize(), + tablet.getValue(finalI, j)); + } + v.addTimestamp(v.getRowSize(), tablet.getTimestamp(finalI)); + return v; + }); + } + } + if (relationalTabletGroup.size() == 1) { + insertRelationalTabletOnce(relationalTabletGroup); + } else { + insertRelationalTabletByGroup(relationalTabletGroup); + } + } + + private void insertRelationalTabletOnce(Map relationalTabletGroup) + throws IoTDBConnectionException, StatementExecutionException { + Map.Entry entry = relationalTabletGroup.entrySet().iterator().next(); + SessionConnection connection = entry.getKey(); + Tablet tablet = entry.getValue(); + TSInsertTabletReq request = genTSInsertTabletReq(tablet, false, false); + request.setWriteToTable(true); + request.setColumnCategories( + tablet.getColumnTypes().stream().map(t -> (byte) t.ordinal()).collect(Collectors.toList())); + try { + connection.insertTablet(request); + } catch (RedirectException e) { + List endPointList = e.getEndPointList(); + Map endPointMap = new HashMap<>(); + for (int i = 0; i < endPointList.size(); i++) { + if (endPointList.get(i) != null) { + endPointMap.put(tablet.getDeviceID(i), endPointList.get(i)); + } + } + endPointMap.forEach(this::handleRedirection); + } catch (IoTDBConnectionException e) { + if (endPointToSessionConnection != null && endPointToSessionConnection.size() > 1) { + // remove the broken session + removeBrokenSessionConnection(connection); + try { + defaultSessionConnection.insertTablet(request); + } catch (RedirectException ignored) { + } + } else { + throw e; + } + } + } + + @SuppressWarnings({ + "squid:S3776" + }) // ignore Cognitive Complexity of methods should not be too high + private void insertRelationalTabletByGroup(Map relationalTabletGroup) + throws IoTDBConnectionException, StatementExecutionException { + List> completableFutures = + relationalTabletGroup.entrySet().stream() + .map( + entry -> { + SessionConnection connection = entry.getKey(); + Tablet subTablet = entry.getValue(); + return CompletableFuture.runAsync( + () -> { + TSInsertTabletReq request = genTSInsertTabletReq(subTablet, false, false); + request.setWriteToTable(true); + request.setColumnCategories( + subTablet.getColumnTypes().stream() + .map(t -> (byte) t.ordinal()) + .collect(Collectors.toList())); + InsertConsumer insertConsumer = + SessionConnection::insertTablet; + try { + insertConsumer.insert(connection, request); + } catch (RedirectException e) { + List endPointList = e.getEndPointList(); + Map endPointMap = new HashMap<>(); + for (int i = 0; i < endPointList.size(); i++) { + if (endPointList.get(i) != null) { + endPointMap.put(subTablet.getDeviceID(i), endPointList.get(i)); + } + } + endPointMap.forEach(this::handleRedirection); + } catch (StatementExecutionException e) { + throw new CompletionException(e); + } catch (IoTDBConnectionException e) { + // remove the broken session + removeBrokenSessionConnection(connection); + try { + insertConsumer.insert(defaultSessionConnection, request); + } catch (IoTDBConnectionException | StatementExecutionException ex) { + throw new CompletionException(ex); + } catch (RedirectException ignored) { + } + } + }, + OPERATION_EXECUTOR); + }) + .collect(Collectors.toList()); + + StringBuilder errMsgBuilder = new StringBuilder(); + for (CompletableFuture completableFuture : completableFutures) { + try { + completableFuture.join(); + } catch (CompletionException completionException) { + Throwable cause = completionException.getCause(); + logger.error("Meet error when async insert!", cause); + if (cause instanceof IoTDBConnectionException) { + throw (IoTDBConnectionException) cause; + } else { + if (errMsgBuilder.length() > 0) { + errMsgBuilder.append(";"); + } + errMsgBuilder.append(cause.getMessage()); + } + } + } + if (errMsgBuilder.length() > 0) { + throw new StatementExecutionException(errMsgBuilder.toString()); + } + } + /** * insert the aligned timeseries data of a device. For each timestamp, the number of measurements * is the same. @@ -2657,15 +2938,15 @@ public void insertAlignedTablet(Tablet tablet, boolean sorted) throws IoTDBConnectionException, StatementExecutionException { TSInsertTabletReq request = genTSInsertTabletReq(tablet, sorted, true); try { - getSessionConnection(tablet.deviceId).insertTablet(request); + getSessionConnection(tablet.getDeviceId()).insertTablet(request); } catch (RedirectException e) { - handleRedirection(tablet.deviceId, e.getEndPoint()); + handleRedirection(tablet.getDeviceId(), e.getEndPoint()); } catch (IoTDBConnectionException e) { if (enableRedirection && !deviceIdToEndpoint.isEmpty() - && deviceIdToEndpoint.get(tablet.deviceId) != null) { - logger.warn(SESSION_CANNOT_CONNECT, deviceIdToEndpoint.get(tablet.deviceId)); - deviceIdToEndpoint.remove(tablet.deviceId); + && deviceIdToEndpoint.get(tablet.getDeviceId()) != null) { + logger.warn(SESSION_CANNOT_CONNECT, deviceIdToEndpoint.get(tablet.getDeviceId())); + deviceIdToEndpoint.remove(tablet.getDeviceId()); // reconnect with default connection try { @@ -2686,15 +2967,18 @@ private TSInsertTabletReq genTSInsertTabletReq(Tablet tablet, boolean sorted, bo TSInsertTabletReq request = new TSInsertTabletReq(); for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { - request.addToMeasurements(measurementSchema.getMeasurementId()); + if (measurementSchema.getMeasurementName() == null) { + throw new IllegalArgumentException("measurement should be non null value"); + } + request.addToMeasurements(measurementSchema.getMeasurementName()); request.addToTypes(measurementSchema.getType().ordinal()); } - request.setPrefixPath(tablet.deviceId); + request.setPrefixPath(tablet.getDeviceId()); request.setIsAligned(isAligned); request.setTimestamps(SessionUtils.getTimeBuffer(tablet)); request.setValues(SessionUtils.getValueBuffer(tablet)); - request.setSize(tablet.rowSize); + request.setSize(tablet.getRowSize()); return request; } @@ -2781,7 +3065,11 @@ private void insertTabletsWithLeaderCache( updateTSInsertTabletsReq(request, entry.getValue(), sorted, isAligned); } - insertByGroup(tabletGroup, SessionConnection::insertTablets); + if (tabletGroup.size() == 1) { + insertOnce(tabletGroup, SessionConnection::insertTablets); + } else { + insertByGroup(tabletGroup, SessionConnection::insertTablets); + } } private TSInsertTabletsReq genTSInsertTabletsReq( @@ -2801,19 +3089,22 @@ private void updateTSInsertTabletsReq( if (!checkSorted(tablet)) { sortTablet(tablet); } - request.addToPrefixPaths(tablet.deviceId); + request.addToPrefixPaths(tablet.getDeviceId()); List measurements = new ArrayList<>(); List dataTypes = new ArrayList<>(); request.setIsAligned(isAligned); for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { - measurements.add(measurementSchema.getMeasurementId()); + if (measurementSchema.getMeasurementName() == null) { + throw new IllegalArgumentException("measurement should be non null value"); + } + measurements.add(measurementSchema.getMeasurementName()); dataTypes.add(measurementSchema.getType().ordinal()); } request.addToMeasurementsList(measurements); request.addToTypesList(dataTypes); request.addToTimestampsList(SessionUtils.getTimeBuffer(tablet)); request.addToValuesList(SessionUtils.getValueBuffer(tablet)); - request.addToSizeList(tablet.rowSize); + request.addToSizeList(tablet.getRowSize()); } // sample some records and judge weather need to add too many null values to convert to tablet. @@ -2864,7 +3155,7 @@ private void convertToTabletAndInsert( } } } - List schemaList = new ArrayList<>(measuremenMap.size()); + List schemaList = new ArrayList<>(measuremenMap.size()); // use measurementType to build schemaList for (Entry> entry : measuremenMap.entrySet()) { schemaList.add(new MeasurementSchema(entry.getKey(), entry.getValue().getLeft())); @@ -2949,11 +3240,11 @@ private void convertToTabletsAndInsert( rowMap.merge(device, 1, Integer::sum); } // device -> schema - Map> schemaMap = new HashMap<>(deviceSize + 1, 1); + Map> schemaMap = new HashMap<>(deviceSize + 1, 1); // use measurementTypeMap to build schemaMap for (Map.Entry>> entry : deviceMeasuremenMap.entrySet()) { - List schemaList = new ArrayList<>(entry.getValue().size() + 1); + List schemaList = new ArrayList<>(entry.getValue().size() + 1); for (Map.Entry> schemaEntry : entry.getValue().entrySet()) { schemaList.add( new MeasurementSchema(schemaEntry.getKey(), schemaEntry.getValue().getLeft())); @@ -2989,7 +3280,7 @@ private void addRecordToTablet( List measurements, List values, Map> allMeasurementMap) { - int row = tablet.rowSize++; + int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp); // tablet without null value if (measurements.size() == allMeasurementMap.size()) { @@ -3219,8 +3510,8 @@ private TSDeleteDataReq genTSDeleteDataReq(List paths, long startTime, l * @return whether the batch has been sorted */ private boolean checkSorted(Tablet tablet) { - for (int i = 1; i < tablet.rowSize; i++) { - if (tablet.timestamps[i] < tablet.timestamps[i - 1]) { + for (int i = 1; i < tablet.getRowSize(); i++) { + if (tablet.getTimestamp(i) < tablet.getTimestamp(i - 1)) { return false; } } @@ -3245,31 +3536,32 @@ public void sortTablet(Tablet tablet) { * so we can insert continuous data in value list to get a better performance */ // sort to get index, and use index to sort value list - Integer[] index = new Integer[tablet.rowSize]; - for (int i = 0; i < tablet.rowSize; i++) { + long[] timestamps = tablet.getTimestamps(); + Object[] values = tablet.getValues(); + BitMap[] bitMaps = tablet.getBitMaps(); + Integer[] index = new Integer[tablet.getRowSize()]; + for (int i = 0; i < tablet.getRowSize(); i++) { index[i] = i; } - Arrays.sort(index, Comparator.comparingLong(o -> tablet.timestamps[o])); - Arrays.sort(tablet.timestamps, 0, tablet.rowSize); + Arrays.sort(index, Comparator.comparingLong(o -> timestamps[o])); + Arrays.sort(timestamps, 0, tablet.getRowSize()); int columnIndex = 0; for (int i = 0; i < tablet.getSchemas().size(); i++) { IMeasurementSchema schema = tablet.getSchemas().get(i); if (schema instanceof MeasurementSchema) { - tablet.values[columnIndex] = sortList(tablet.values[columnIndex], schema.getType(), index); - if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { - tablet.bitMaps[columnIndex] = sortBitMap(tablet.bitMaps[columnIndex], index); + values[columnIndex] = sortList(values[columnIndex], schema.getType(), index); + if (bitMaps != null && bitMaps[columnIndex] != null) { + bitMaps[columnIndex] = sortBitMap(bitMaps[columnIndex], index); } columnIndex++; } else { int measurementSize = schema.getSubMeasurementsList().size(); for (int j = 0; j < measurementSize; j++) { - tablet.values[columnIndex] = + values[columnIndex] = sortList( - tablet.values[columnIndex], - schema.getSubMeasurementsTSDataTypeList().get(j), - index); - if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { - tablet.bitMaps[columnIndex] = sortBitMap(tablet.bitMaps[columnIndex], index); + values[columnIndex], schema.getSubMeasurementsTSDataTypeList().get(j), index); + if (bitMaps != null && bitMaps[columnIndex] != null) { + bitMaps[columnIndex] = sortBitMap(bitMaps[columnIndex], index); } columnIndex++; } @@ -3295,13 +3587,19 @@ private Object sortList(Object valueList, TSDataType dataType, Integer[] index) } return sortedValues; case INT32: - case DATE: int[] intValues = (int[]) valueList; int[] sortedIntValues = new int[intValues.length]; for (int i = 0; i < index.length; i++) { sortedIntValues[i] = intValues[index[i]]; } return sortedIntValues; + case DATE: + LocalDate[] date = (LocalDate[]) valueList; + LocalDate[] sortedDateValues = new LocalDate[date.length]; + for (int i = 0; i < index.length; i++) { + sortedDateValues[i] = date[index[i]]; + } + return sortedDateValues; case INT64: case TIMESTAMP: long[] longValues = (long[]) valueList; @@ -3439,8 +3737,6 @@ public void createSchemaTemplate( * @param encodings the encoding of each measurement, only the first one in each nested list * matters as above * @param compressors the compressor of each measurement - * @throws IoTDBConnectionException - * @throws StatementExecutionException * @deprecated */ @Override @@ -3737,8 +4033,32 @@ public void createTimeseriesUsingSchemaTemplate(List devicePathList) defaultSessionConnection.createTimeseriesUsingSchemaTemplate(request); } + private void insertOnce( + Map insertGroup, InsertConsumer insertConsumer) + throws IoTDBConnectionException, StatementExecutionException { + Map.Entry entry = insertGroup.entrySet().iterator().next(); + SessionConnection connection = entry.getKey(); + T insertReq = entry.getValue(); + try { + insertConsumer.insert(connection, insertReq); + } catch (RedirectException e) { + e.getDeviceEndPointMap().forEach(this::handleRedirection); + } catch (IoTDBConnectionException e) { + if (endPointToSessionConnection != null && endPointToSessionConnection.size() > 1) { + // remove the broken session + removeBrokenSessionConnection(connection); + try { + insertConsumer.insert(defaultSessionConnection, insertReq); + } catch (RedirectException ignored) { + } + } else { + throw e; + } + } + } + /** - * @param recordsGroup connection to record map + * @param insertGroup connection to request map * @param insertConsumer insert function * @param *
    @@ -3746,26 +4066,23 @@ public void createTimeseriesUsingSchemaTemplate(List devicePathList) *
  • {@link TSInsertStringRecordsReq} *
  • {@link TSInsertTabletsReq} *
- * - * @throws IoTDBConnectionException - * @throws StatementExecutionException */ @SuppressWarnings({ "squid:S3776" }) // ignore Cognitive Complexity of methods should not be too high private void insertByGroup( - Map recordsGroup, InsertConsumer insertConsumer) + Map insertGroup, InsertConsumer insertConsumer) throws IoTDBConnectionException, StatementExecutionException { List> completableFutures = - recordsGroup.entrySet().stream() + insertGroup.entrySet().stream() .map( entry -> { SessionConnection connection = entry.getKey(); - T recordsReq = entry.getValue(); + T insertReq = entry.getValue(); return CompletableFuture.runAsync( () -> { try { - insertConsumer.insert(connection, recordsReq); + insertConsumer.insert(connection, insertReq); } catch (RedirectException e) { e.getDeviceEndPointMap().forEach(this::handleRedirection); } catch (StatementExecutionException e) { @@ -3774,7 +4091,7 @@ private void insertByGroup( // remove the broken session removeBrokenSessionConnection(connection); try { - insertConsumer.insert(defaultSessionConnection, recordsReq); + insertConsumer.insert(defaultSessionConnection, insertReq); } catch (IoTDBConnectionException | StatementExecutionException ex) { throw new CompletionException(ex); } catch (RedirectException ignored) { @@ -3795,6 +4112,9 @@ private void insertByGroup( if (cause instanceof IoTDBConnectionException) { throw (IoTDBConnectionException) cause; } else { + if (errMsgBuilder.length() > 0) { + errMsgBuilder.append(";"); + } errMsgBuilder.append(cause.getMessage()); } } @@ -3835,60 +4155,25 @@ public TSConnectionInfoResp fetchAllConnections() throws IoTDBConnectionExceptio return defaultSessionConnection.fetchAllConnections(); } - public static class Builder { - private String host = SessionConfig.DEFAULT_HOST; - private int rpcPort = SessionConfig.DEFAULT_PORT; - private String username = SessionConfig.DEFAULT_USER; - private String pw = SessionConfig.DEFAULT_PASSWORD; - private int fetchSize = SessionConfig.DEFAULT_FETCH_SIZE; - private ZoneId zoneId = null; - private int thriftDefaultBufferSize = SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY; - private int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; - // this field only take effect in write request, nothing to do with any other type requests, - // like query, load and so on. - // if set to true, it means that we may redirect the write request to its corresponding leader - // if set to false, it means that we will only send write request to first available DataNode(it - // may be changed while current DataNode is not available, for example, we may retry to connect - // to another available DataNode) - // so even if enableRedirection is set to false, we may also send write request to another - // datanode while encountering retriable errors in current DataNode - private boolean enableRedirection = SessionConfig.DEFAULT_REDIRECTION_MODE; - private boolean enableRecordsAutoConvertTablet = - SessionConfig.DEFAULT_RECORDS_AUTO_CONVERT_TABLET; - private Version version = SessionConfig.DEFAULT_VERSION; - private long timeOut = SessionConfig.DEFAULT_QUERY_TIME_OUT; - - // set to true, means that we will start a background thread to fetch all available (Status is - // not Removing) datanodes in cluster, and these available nodes will be used in retrying stage - private boolean enableAutoFetch = SessionConfig.DEFAULT_ENABLE_AUTO_FETCH; - - private boolean useSSL = false; - private String trustStore; - private String trustStorePwd; - - // max retry count, if set to 0, means that we won't do any retry - // we can use any available DataNodes(fetched in background thread if enableAutoFetch is true, - // or nodeUrls user specified) to retry, even if enableRedirection is false - private int maxRetryCount = SessionConfig.MAX_RETRY_COUNT; - - private long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; + protected void changeDatabase(String database) { + this.database = database; + } - public Builder useSSL(boolean useSSL) { - this.useSSL = useSSL; - return this; - } + public String getDatabase() { + return database; + } - public Builder trustStore(String keyStore) { - this.trustStore = keyStore; - return this; - } + protected void changeSqlDialect(String sqlDialect) { + this.sqlDialect = sqlDialect; + // clean database to avoid misuse of it between different SqlDialect + this.database = null; + } - public Builder trustStorePwd(String keyStorePwd) { - this.trustStorePwd = keyStorePwd; - return this; - } + public String getSqlDialect() { + return sqlDialect; + } - private List nodeUrls = null; + public static class Builder extends AbstractSessionBuilder { public Builder host(String host) { this.host = host; @@ -3970,14 +4255,38 @@ public Builder retryIntervalInMs(long retryIntervalInMs) { return this; } + public Builder sqlDialect(String sqlDialect) { + this.sqlDialect = sqlDialect; + return this; + } + + public Builder database(String database) { + this.database = database; + return this; + } + + public Builder useSSL(boolean useSSL) { + this.useSSL = useSSL; + return this; + } + + public Builder trustStore(String keyStore) { + this.trustStore = keyStore; + return this; + } + + public Builder trustStorePwd(String keyStorePwd) { + this.trustStorePwd = keyStorePwd; + return this; + } + public Session build() { if (nodeUrls != null && (!SessionConfig.DEFAULT_HOST.equals(host) || rpcPort != SessionConfig.DEFAULT_PORT)) { throw new IllegalArgumentException( "You should specify either nodeUrls or (host + rpcPort), but not both"); } - Session newSession = new Session(this); - return newSession; + return new Session(this); } } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 4a8c3a36ce5be..491071f093aa7 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -29,6 +29,7 @@ import org.apache.iotdb.rpc.RedirectException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.IClientRPCService; import org.apache.iotdb.service.rpc.thrift.TCreateTimeseriesUsingSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSAggregationQueryReq; @@ -71,6 +72,7 @@ import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,8 +84,13 @@ import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; +import static org.apache.iotdb.session.Session.TABLE; +import static org.apache.iotdb.session.Session.TREE; + @SuppressWarnings("java:S2142") public class SessionConnection { @@ -106,11 +113,20 @@ public class SessionConnection { private final long retryIntervalInMs; + private String sqlDialect; + + private String database; + + // ms is 1_000, us is 1_000_000, ns is 1_000_000_000 + private int timeFactor = 1_000; + // TestOnly - public SessionConnection() { + public SessionConnection(String sqlDialect) { availableNodes = Collections::emptyList; this.maxRetryCount = Math.max(0, SessionConfig.MAX_RETRY_COUNT); this.retryIntervalInMs = Math.max(0, SessionConfig.RETRY_INTERVAL_IN_MS); + this.sqlDialect = sqlDialect; + database = null; } public SessionConnection( @@ -119,7 +135,9 @@ public SessionConnection( ZoneId zoneId, Supplier> availableNodes, int maxRetryCount, - long retryIntervalInMs) + long retryIntervalInMs, + String sqlDialect, + String database) throws IoTDBConnectionException { this.session = session; this.endPoint = endPoint; @@ -128,6 +146,8 @@ public SessionConnection( this.availableNodes = availableNodes; this.maxRetryCount = Math.max(0, maxRetryCount); this.retryIntervalInMs = Math.max(0, retryIntervalInMs); + this.sqlDialect = sqlDialect; + this.database = database; try { init(endPoint, session.useSSL, session.trustStore, session.trustStorePwd); } catch (StatementExecutionException e) { @@ -142,7 +162,9 @@ public SessionConnection( ZoneId zoneId, Supplier> availableNodes, int maxRetryCount, - long retryIntervalInMs) + long retryIntervalInMs, + String sqlDialect, + String database) throws IoTDBConnectionException { this.session = session; this.zoneId = zoneId == null ? ZoneId.systemDefault() : zoneId; @@ -150,6 +172,8 @@ public SessionConnection( this.availableNodes = availableNodes; this.maxRetryCount = Math.max(0, maxRetryCount); this.retryIntervalInMs = Math.max(0, retryIntervalInMs); + this.sqlDialect = sqlDialect; + this.database = database; initClusterConn(); } @@ -191,12 +215,16 @@ private void init(TEndPoint endPoint, boolean useSSL, String trustStore, String openReq.setPassword(session.password); openReq.setZoneId(zoneId.toString()); openReq.putToConfiguration("version", session.version.toString()); + openReq.putToConfiguration("sql_dialect", sqlDialect); + if (database != null) { + openReq.putToConfiguration("db", database); + } try { TSOpenSessionResp openResp = client.openSession(openReq); RpcUtils.verifySuccess(openResp.getStatus()); - + this.timeFactor = RpcUtils.getTimeFactor(openResp); if (Session.protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { logger.warn( "Protocol differ, Client version is {}}, but Server version is {}", @@ -262,23 +290,14 @@ protected IClientRPCService.Iface getClient() { protected void setTimeZone(String zoneId) throws StatementExecutionException, IoTDBConnectionException { - TSSetTimeZoneReq req = new TSSetTimeZoneReq(sessionId, zoneId); - TSStatus resp; - try { - resp = client.setTimeZone(req); - } catch (TException e) { - if (reconnect()) { - try { - req.setSessionId(sessionId); - resp = client.setTimeZone(req); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } - RpcUtils.verifySuccess(resp); + final TSStatus status = + callWithRetryAndReconnect( + () -> { + TSSetTimeZoneReq req = new TSSetTimeZoneReq(sessionId, zoneId); + return client.setTimeZone(req); + }) + .getResult(); + RpcUtils.verifySuccess(status); setTimeZoneOfSession(zoneId); } @@ -295,93 +314,54 @@ protected String getTimeZone() { protected void setStorageGroup(String storageGroup) throws IoTDBConnectionException, StatementExecutionException { - try { - RpcUtils.verifySuccess(client.setStorageGroup(sessionId, storageGroup)); - } catch (TException e) { - if (reconnect()) { - try { - RpcUtils.verifySuccess(client.setStorageGroup(sessionId, storageGroup)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect(() -> client.setStorageGroup(sessionId, storageGroup)) + .getResult(); + RpcUtils.verifySuccess(status); } protected void deleteStorageGroups(List storageGroups) throws IoTDBConnectionException, StatementExecutionException { - try { - RpcUtils.verifySuccess(client.deleteStorageGroups(sessionId, storageGroups)); - } catch (TException e) { - if (reconnect()) { - try { - RpcUtils.verifySuccess(client.deleteStorageGroups(sessionId, storageGroups)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect(() -> client.deleteStorageGroups(sessionId, storageGroups)) + .getResult(); + RpcUtils.verifySuccess(status); } protected void createTimeseries(TSCreateTimeseriesReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.createTimeseries(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.createTimeseries(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.createTimeseries(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void createAlignedTimeseries(TSCreateAlignedTimeseriesReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.createAlignedTimeseries(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.createAlignedTimeseries(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.createAlignedTimeseries(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void createMultiTimeseries(TSCreateMultiTimeseriesReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.createMultiTimeseries(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.createMultiTimeseries(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.createMultiTimeseries(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected boolean checkTimeseriesExists(String path, long timeout) @@ -406,26 +386,23 @@ protected SessionDataSet executeQueryStatement(String sql, long timeout) TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionId, sql, statementId); execReq.setFetchSize(session.fetchSize); execReq.setTimeout(timeout); - TSExecuteStatementResp execResp; - try { - execReq.setEnableRedirectQuery(enableRedirect); - execResp = client.executeQueryStatementV2(execReq); + execReq.setEnableRedirectQuery(enableRedirect); + + RetryResult result = + callWithRetryAndReconnect( + () -> { + execReq.setSessionId(sessionId); + execReq.setStatementId(statementId); + return client.executeQueryStatementV2(execReq); + }, + TSExecuteStatementResp::getStatus); + TSExecuteStatementResp execResp = result.getResult(); + if (result.getRetryAttempts() == 0) { RpcUtils.verifySuccessWithRedirection(execResp.getStatus()); - } catch (TException e) { - if (reconnect()) { - try { - execReq.setSessionId(sessionId); - execReq.setStatementId(statementId); - execResp = client.executeQueryStatementV2(execReq); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + } else { + RpcUtils.verifySuccess(execResp.getStatus()); } - RpcUtils.verifySuccess(execResp.getStatus()); return new SessionDataSet( sql, execResp.getColumns(), @@ -440,61 +417,36 @@ protected SessionDataSet executeQueryStatement(String sql, long timeout) timeout, execResp.moreData, session.fetchSize, - zoneId); + zoneId, + timeFactor, + execResp.isSetTableModel() && execResp.isTableModel(), + execResp.getColumnIndex2TsBlockColumnIndexList()); } protected void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { - TSExecuteStatementReq request = new TSExecuteStatementReq(sessionId, sql, statementId); - - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = executeNonQueryStatementInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - RpcUtils.verifySuccess(status); - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerify(() -> executeNonQueryStatementInternal(request)); } private TSStatus executeNonQueryStatementInternal(TSExecuteStatementReq request) throws TException { request.setSessionId(sessionId); request.setStatementId(statementId); - return client.executeUpdateStatementV2(request).status; + TSExecuteStatementResp resp = client.executeUpdateStatementV2(request); + if (resp.isSetDatabase()) { + String dbName = resp.getDatabase(); + session.changeDatabase(dbName); + this.database = dbName; + } + if (resp.isSetTableModel()) { + String sqlDialect = resp.tableModel ? TABLE : TREE; + if (!sqlDialect.equalsIgnoreCase(this.sqlDialect)) { + session.changeSqlDialect(sqlDialect); + this.sqlDialect = sqlDialect; + } + } + return resp.status; } protected SessionDataSet executeRawDataQuery( @@ -504,26 +456,24 @@ protected SessionDataSet executeRawDataQuery( new TSRawDataQueryReq(sessionId, paths, startTime, endTime, statementId); execReq.setFetchSize(session.fetchSize); execReq.setTimeout(timeOut); - TSExecuteStatementResp execResp; - try { - execReq.setEnableRedirectQuery(enableRedirect); - execResp = client.executeRawDataQueryV2(execReq); + execReq.setEnableRedirectQuery(enableRedirect); + + RetryResult result = + callWithRetryAndReconnect( + () -> { + execReq.setSessionId(sessionId); + execReq.setStatementId(statementId); + return client.executeRawDataQueryV2(execReq); + }, + TSExecuteStatementResp::getStatus); + + TSExecuteStatementResp execResp = result.getResult(); + if (result.getRetryAttempts() == 0) { RpcUtils.verifySuccessWithRedirection(execResp.getStatus()); - } catch (TException e) { - if (reconnect()) { - try { - execReq.setSessionId(sessionId); - execReq.setStatementId(statementId); - execResp = client.executeRawDataQueryV2(execReq); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + } else { + RpcUtils.verifySuccess(execResp.getStatus()); } - RpcUtils.verifySuccess(execResp.getStatus()); return new SessionDataSet( "", execResp.getColumns(), @@ -536,7 +486,10 @@ protected SessionDataSet executeRawDataQuery( execResp.queryResult, execResp.isIgnoreTimeStamp(), execResp.moreData, - zoneId); + zoneId, + timeFactor, + execResp.isSetTableModel() && execResp.isTableModel(), + execResp.getColumnIndex2TsBlockColumnIndexList()); } protected Pair executeLastDataQueryForOneDevice( @@ -548,28 +501,28 @@ protected Pair executeLastDataQueryForOneDevice( req.setEnableRedirectQuery(enableRedirect); req.setLegalPathNodes(isLegalPathNodes); req.setTimeout(timeOut); - TSExecuteStatementResp tsExecuteStatementResp = null; TEndPoint redirectedEndPoint = null; - try { - tsExecuteStatementResp = client.executeFastLastDataQueryForOneDeviceV2(req); - RpcUtils.verifySuccessWithRedirection(tsExecuteStatementResp.getStatus()); - } catch (RedirectException e) { - redirectedEndPoint = e.getEndPoint(); - } catch (TException e) { - if (reconnect()) { - try { - req.setSessionId(sessionId); - req.setStatementId(statementId); - tsExecuteStatementResp = client.executeFastLastDataQueryForOneDeviceV2(req); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); + + RetryResult result = + callWithRetryAndReconnect( + () -> { + req.setSessionId(sessionId); + req.setStatementId(statementId); + return client.executeFastLastDataQueryForOneDeviceV2(req); + }, + TSExecuteStatementResp::getStatus); + + TSExecuteStatementResp tsExecuteStatementResp = result.getResult(); + if (result.getRetryAttempts() == 0) { + try { + RpcUtils.verifySuccessWithRedirection(tsExecuteStatementResp.getStatus()); + } catch (RedirectException e) { + redirectedEndPoint = e.getEndPoint(); } + } else { + RpcUtils.verifySuccess(tsExecuteStatementResp.getStatus()); } - RpcUtils.verifySuccess(tsExecuteStatementResp.getStatus()); return new Pair<>( new SessionDataSet( "", @@ -583,7 +536,10 @@ protected Pair executeLastDataQueryForOneDevice( tsExecuteStatementResp.queryResult, tsExecuteStatementResp.isIgnoreTimeStamp(), tsExecuteStatementResp.moreData, - zoneId), + zoneId, + timeFactor, + tsExecuteStatementResp.isSetTableModel() && tsExecuteStatementResp.isTableModel(), + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()), redirectedEndPoint); } @@ -594,25 +550,23 @@ protected SessionDataSet executeLastDataQuery(List paths, long time, lon tsLastDataQueryReq.setFetchSize(session.fetchSize); tsLastDataQueryReq.setEnableRedirectQuery(enableRedirect); tsLastDataQueryReq.setTimeout(timeOut); - TSExecuteStatementResp tsExecuteStatementResp; - try { - tsExecuteStatementResp = client.executeLastDataQueryV2(tsLastDataQueryReq); + + RetryResult result = + callWithRetryAndReconnect( + () -> { + tsLastDataQueryReq.setSessionId(sessionId); + tsLastDataQueryReq.setStatementId(statementId); + return client.executeLastDataQueryV2(tsLastDataQueryReq); + }, + TSExecuteStatementResp::getStatus); + final TSExecuteStatementResp tsExecuteStatementResp = result.getResult(); + + if (result.getRetryAttempts() == 0) { RpcUtils.verifySuccessWithRedirection(tsExecuteStatementResp.getStatus()); - } catch (TException e) { - if (reconnect()) { - try { - tsLastDataQueryReq.setSessionId(sessionId); - tsLastDataQueryReq.setStatementId(statementId); - tsExecuteStatementResp = client.executeLastDataQueryV2(tsLastDataQueryReq); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + } else { + RpcUtils.verifySuccess(tsExecuteStatementResp.getStatus()); } - RpcUtils.verifySuccess(tsExecuteStatementResp.getStatus()); return new SessionDataSet( "", tsExecuteStatementResp.getColumns(), @@ -625,7 +579,10 @@ protected SessionDataSet executeLastDataQuery(List paths, long time, lon tsExecuteStatementResp.queryResult, tsExecuteStatementResp.isIgnoreTimeStamp(), tsExecuteStatementResp.moreData, - zoneId); + zoneId, + timeFactor, + tsExecuteStatementResp.isSetTableModel() && tsExecuteStatementResp.isTableModel(), + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()); } protected SessionDataSet executeAggregationQuery( @@ -676,25 +633,22 @@ protected SessionDataSet executeAggregationQuery( private SessionDataSet executeAggregationQuery(TSAggregationQueryReq tsAggregationQueryReq) throws StatementExecutionException, IoTDBConnectionException, RedirectException { - TSExecuteStatementResp tsExecuteStatementResp; - try { - tsExecuteStatementResp = client.executeAggregationQueryV2(tsAggregationQueryReq); + RetryResult result = + callWithRetryAndReconnect( + () -> { + tsAggregationQueryReq.setSessionId(sessionId); + tsAggregationQueryReq.setStatementId(statementId); + return client.executeAggregationQueryV2(tsAggregationQueryReq); + }, + TSExecuteStatementResp::getStatus); + + TSExecuteStatementResp tsExecuteStatementResp = result.getResult(); + if (result.getRetryAttempts() == 0) { RpcUtils.verifySuccessWithRedirection(tsExecuteStatementResp.getStatus()); - } catch (TException e) { - if (reconnect()) { - try { - tsAggregationQueryReq.setSessionId(sessionId); - tsAggregationQueryReq.setStatementId(statementId); - tsExecuteStatementResp = client.executeAggregationQuery(tsAggregationQueryReq); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + } else { + RpcUtils.verifySuccess(tsExecuteStatementResp.getStatus()); } - RpcUtils.verifySuccess(tsExecuteStatementResp.getStatus()); return new SessionDataSet( "", tsExecuteStatementResp.getColumns(), @@ -707,7 +661,10 @@ private SessionDataSet executeAggregationQuery(TSAggregationQueryReq tsAggregati tsExecuteStatementResp.queryResult, tsExecuteStatementResp.isIgnoreTimeStamp(), tsExecuteStatementResp.moreData, - zoneId); + zoneId, + timeFactor, + tsExecuteStatementResp.isSetTableModel() && tsExecuteStatementResp.isTableModel(), + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()); } private TSAggregationQueryReq createAggregationQueryReq( @@ -721,52 +678,7 @@ private TSAggregationQueryReq createAggregationQueryReq( protected void insertRecord(TSInsertRecordReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertRecordInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirection(status); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirection(() -> insertRecordInternal(request)); } private TSStatus insertRecordInternal(TSInsertRecordReq request) throws TException { @@ -776,52 +688,7 @@ private TSStatus insertRecordInternal(TSInsertRecordReq request) throws TExcepti protected void insertRecord(TSInsertStringRecordReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertRecordInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirection(status); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirection(() -> insertRecordInternal(request)); } private TSStatus insertRecordInternal(TSInsertStringRecordReq request) throws TException { @@ -831,52 +698,8 @@ private TSStatus insertRecordInternal(TSInsertStringRecordReq request) throws TE protected void insertRecords(TSInsertRecordsReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertRecordsInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirectionForMultiDevices(status, request.getPrefixPaths()); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirectionForMultipleDevices( + () -> insertRecordsInternal(request), request::getPrefixPaths); } private TSStatus insertRecordsInternal(TSInsertRecordsReq request) throws TException { @@ -886,53 +709,8 @@ private TSStatus insertRecordsInternal(TSInsertRecordsReq request) throws TExcep protected void insertRecords(TSInsertStringRecordsReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertRecordsInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirectionForMultiDevices(status, request.getPrefixPaths()); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirectionForMultipleDevices( + () -> insertRecordsInternal(request), request::getPrefixPaths); } private TSStatus insertRecordsInternal(TSInsertStringRecordsReq request) throws TException { @@ -942,53 +720,7 @@ private TSStatus insertRecordsInternal(TSInsertStringRecordsReq request) throws protected void insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertRecordsOfOneDeviceInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirection(status); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirection(() -> insertRecordsOfOneDeviceInternal(request)); } private TSStatus insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq request) @@ -999,53 +731,7 @@ private TSStatus insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq protected void insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertStringRecordsOfOneDeviceInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirection(status); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirection(() -> insertStringRecordsOfOneDeviceInternal(request)); } private TSStatus insertStringRecordsOfOneDeviceInternal( @@ -1054,57 +740,48 @@ private TSStatus insertStringRecordsOfOneDeviceInternal( return client.insertStringRecordsOfOneDevice(request); } - protected void insertTablet(TSInsertTabletReq request) - throws IoTDBConnectionException, StatementExecutionException, RedirectException { + private void callWithRetryAndVerifyWithRedirectionForMultipleDevices( + TFunction function, Supplier> pathSupplier) + throws StatementExecutionException, RedirectException, IoTDBConnectionException { + RetryResult result = callWithRetry(function); - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertTabletInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirection(status); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; + TSStatus status = result.getResult(); + if (status != null) { + if (result.getRetryAttempts() == 0) { + RpcUtils.verifySuccessWithRedirectionForMultiDevices(status, pathSupplier.get()); + } else { + RpcUtils.verifySuccess(status); } + } else if (result.getException() != null) { + throw new IoTDBConnectionException(result.getException()); + } else { + throw new IoTDBConnectionException(logForReconnectionFailure()); } + } + + private void callWithRetryAndVerifyWithRedirection(TFunction function) + throws StatementExecutionException, RedirectException, IoTDBConnectionException { + RetryResult result = callWithRetry(function); + TSStatus status = result.getResult(); if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); + if (result.getRetryAttempts() == 0) { + RpcUtils.verifySuccessWithRedirection(status); + } else { + RpcUtils.verifySuccess(status); + } + } else if (result.getException() != null) { + throw new IoTDBConnectionException(result.getException()); } else { throw new IoTDBConnectionException(logForReconnectionFailure()); } } + protected void insertTablet(TSInsertTabletReq request) + throws IoTDBConnectionException, StatementExecutionException, RedirectException { + callWithRetryAndVerifyWithRedirection(() -> insertTabletInternal(request)); + } + private TSStatus insertTabletInternal(TSInsertTabletReq request) throws TException { request.setSessionId(sessionId); return client.insertTablet(request); @@ -1112,53 +789,8 @@ private TSStatus insertTabletInternal(TSInsertTabletReq request) throws TExcepti protected void insertTablets(TSInsertTabletsReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { - - TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } - try { - status = insertTabletsInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - if (i == 0) { - // first time succeed, take account for redirection info - RpcUtils.verifySuccessWithRedirectionForMultiDevices(status, request.getPrefixPaths()); - } else { - // if it's retry, just ignore redirection info - RpcUtils.verifySuccess(status); - } - return; - } catch (TException e) { - // all network exception need retry until reaching maxRetryCount - lastTException = e; - } - } - - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + callWithRetryAndVerifyWithRedirectionForMultipleDevices( + () -> insertTabletsInternal(request), request::getPrefixPaths); } private TSStatus insertTabletsInternal(TSInsertTabletsReq request) throws TException { @@ -1168,10 +800,31 @@ private TSStatus insertTabletsInternal(TSInsertTabletsReq request) throws TExcep protected void deleteTimeseries(List paths) throws IoTDBConnectionException, StatementExecutionException { + callWithRetryAndVerify(() -> client.deleteTimeseries(sessionId, paths)); + } + public void deleteData(TSDeleteDataReq request) + throws IoTDBConnectionException, StatementExecutionException { + callWithRetryAndVerify(() -> deleteDataInternal(request)); + } + + private void callWithRetryAndVerify(TFunction rpc) + throws IoTDBConnectionException, StatementExecutionException { + RetryResult result = callWithRetry(rpc); + if (result.getResult() != null) { + RpcUtils.verifySuccess(result.getResult()); + } else if (result.getException() != null) { + throw new IoTDBConnectionException(result.getException()); + } else { + throw new IoTDBConnectionException(logForReconnectionFailure()); + } + } + + private RetryResult callWithRetry(TFunction rpc) { TException lastTException = null; TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { + int i; + for (i = 0; i <= maxRetryCount; i++) { if (i > 0) { // re-init the TException and TSStatus lastTException = null; @@ -1180,7 +833,13 @@ protected void deleteTimeseries(List paths) try { TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); } catch (InterruptedException e) { - // just ignore + Thread.currentThread().interrupt(); + logger.warn( + "Thread {} was interrupted during retry {} with wait time {} ms. Exiting retry loop.", + Thread.currentThread().getName(), + i, + retryIntervalInMs); + break; } if (!reconnect()) { // reconnect failed, just continue to make another retry. @@ -1188,72 +847,89 @@ protected void deleteTimeseries(List paths) } } try { - status = client.deleteTimeseries(sessionId, paths); + status = rpc.run(); // need retry if (status.isSetNeedRetry() && status.isNeedRetry()) { continue; } - // succeed or don't need to retry - RpcUtils.verifySuccess(status); - return; + break; } catch (TException e) { // all network exception need retry until reaching maxRetryCount lastTException = e; } } - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } + return new RetryResult<>(status, lastTException, i); } - public void deleteData(TSDeleteDataReq request) - throws IoTDBConnectionException, StatementExecutionException { + private RetryResult callWithRetryAndReconnect(TFunction rpc) { + return callWithRetryAndReconnect( + rpc, + status -> status.isSetNeedRetry() && status.isNeedRetry(), + status -> status.getCode() == TSStatusCode.PLAN_FAILED_NETWORK_PARTITION.getStatusCode()); + } + private RetryResult callWithRetryAndReconnect( + TFunction rpc, Function statusGetter) { + return callWithRetryAndReconnect( + rpc, + t -> { + final TSStatus status = statusGetter.apply(t); + return status.isSetNeedRetry() && status.isNeedRetry(); + }, + t -> + statusGetter.apply(t).getCode() + == TSStatusCode.PLAN_FAILED_NETWORK_PARTITION.getStatusCode()); + } + + /** reconnect if the remote datanode is unreachable retry if the status is set to needRetry */ + private RetryResult callWithRetryAndReconnect( + TFunction rpc, Predicate shouldRetry, Predicate forceReconnect) { TException lastTException = null; - TSStatus status = null; - for (int i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - // re-init the TException and TSStatus - lastTException = null; - status = null; - // not first time, we need to sleep and then reconnect - try { - TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); - } catch (InterruptedException e) { - // just ignore - } - if (!reconnect()) { - // reconnect failed, just continue to make another retry. - continue; - } - } + T result = null; + int retryAttempt; + int maxRetryCountRead = 10; + for (retryAttempt = 0; retryAttempt <= maxRetryCountRead; retryAttempt++) { + // 1. try to execute the rpc try { - status = deleteDataInternal(request); - // need retry - if (status.isSetNeedRetry() && status.isNeedRetry()) { - continue; - } - // succeed or don't need to retry - RpcUtils.verifySuccess(status); - return; + result = rpc.run(); + lastTException = null; } catch (TException e) { - // all network exception need retry until reaching maxRetryCount + result = null; lastTException = e; } - } - if (status != null) { - RpcUtils.verifySuccess(status); - } else if (lastTException != null) { - throw new IoTDBConnectionException(lastTException); - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); + // success, return immediately + if (result != null && !shouldRetry.test(result)) { + return new RetryResult<>(result, null, retryAttempt); + } + + logger.debug( + "Retry attempt #{}, result {}, exception {}", retryAttempt, result, lastTException); + // prepare for the next retry + if (lastTException != null + || !availableNodes.get().contains(this.endPoint) + || (result != null && forceReconnect.test(result))) { + // 1. the current datanode is unreachable (TException) + // 2. the current datanode is partitioned with other nodes (not in availableNodes) + // 3. asymmetric network partition + logger.debug("Retry attempt #{}, Reconnecting to other datanode", retryAttempt); + reconnect(); + } + try { + TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn( + "Thread {} was interrupted during retry {} with wait time {} ms. Exiting retry loop.", + Thread.currentThread().getName(), + retryAttempt, + retryIntervalInMs); + break; + } } + + return new RetryResult<>(result, lastTException, retryAttempt); } private TSStatus deleteDataInternal(TSDeleteDataReq request) throws TException { @@ -1263,116 +939,74 @@ private TSStatus deleteDataInternal(TSDeleteDataReq request) throws TException { protected void testInsertRecord(TSInsertStringRecordReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.testInsertStringRecord(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.testInsertStringRecord(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.testInsertStringRecord(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void testInsertRecord(TSInsertRecordReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.testInsertRecord(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.testInsertRecord(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.testInsertRecord(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } public void testInsertRecords(TSInsertStringRecordsReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.testInsertStringRecords(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.testInsertStringRecords(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.testInsertStringRecords(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } public void testInsertRecords(TSInsertRecordsReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.testInsertRecords(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.testInsertRecords(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.testInsertRecords(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void testInsertTablet(TSInsertTabletReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.testInsertTablet(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.testInsertTablet(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.testInsertTablet(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void testInsertTablets(TSInsertTabletsReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.testInsertTablets(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.testInsertTablets(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.testInsertTablets(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } @SuppressWarnings({ @@ -1425,189 +1059,124 @@ private boolean reconnect() { protected void createSchemaTemplate(TSCreateSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.createSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.createSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.createSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void appendSchemaTemplate(TSAppendSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.appendSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.appendSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.appendSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void pruneSchemaTemplate(TSPruneSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.pruneSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.pruneSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.pruneSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req) throws StatementExecutionException, IoTDBConnectionException { - TSQueryTemplateResp execResp; - req.setSessionId(sessionId); - try { - execResp = client.querySchemaTemplate(req); - RpcUtils.verifySuccess(execResp.getStatus()); - } catch (TException e) { - if (reconnect()) { - try { - execResp = client.querySchemaTemplate(req); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } - + final TSQueryTemplateResp execResp = + callWithRetryAndReconnect( + () -> { + req.setSessionId(sessionId); + return client.querySchemaTemplate(req); + }, + TSQueryTemplateResp::getStatus) + .getResult(); RpcUtils.verifySuccess(execResp.getStatus()); return execResp; } protected void setSchemaTemplate(TSSetSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.setSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.setSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.setSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void unsetSchemaTemplate(TSUnsetSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.unsetSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.unsetSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.unsetSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void dropSchemaTemplate(TSDropSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.dropSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.dropSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.dropSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected void createTimeseriesUsingSchemaTemplate( TCreateTimeseriesUsingSchemaTemplateReq request) throws IoTDBConnectionException, StatementExecutionException { - request.setSessionId(sessionId); - try { - RpcUtils.verifySuccess(client.createTimeseriesUsingSchemaTemplate(request)); - } catch (TException e) { - if (reconnect()) { - try { - request.setSessionId(sessionId); - RpcUtils.verifySuccess(client.createTimeseriesUsingSchemaTemplate(request)); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(MSG_RECONNECTION_FAIL); - } - } + final TSStatus status = + callWithRetryAndReconnect( + () -> { + request.setSessionId(sessionId); + return client.createTimeseriesUsingSchemaTemplate(request); + }) + .getResult(); + RpcUtils.verifySuccess(status); } protected TSBackupConfigurationResp getBackupConfiguration() throws IoTDBConnectionException, StatementExecutionException { - TSBackupConfigurationResp execResp; - try { - execResp = client.getBackupConfiguration(); - RpcUtils.verifySuccess(execResp.getStatus()); - } catch (TException e) { - if (reconnect()) { - try { - execResp = client.getBackupConfiguration(); - RpcUtils.verifySuccess(execResp.getStatus()); - } catch (TException tException) { - throw new IoTDBConnectionException(tException); - } - } else { - throw new IoTDBConnectionException(logForReconnectionFailure()); - } - } + final TSBackupConfigurationResp execResp = + callWithRetryAndReconnect( + () -> client.getBackupConfiguration(), TSBackupConfigurationResp::getStatus) + .getResult(); + RpcUtils.verifySuccess(execResp.getStatus()); return execResp; } - public TSConnectionInfoResp fetchAllConnections() throws IoTDBConnectionException { + private RetryResult callWithReconnect(TFunction supplier) + throws IoTDBConnectionException { + T ret; try { - return client.fetchAllConnectionsInfo(); + ret = supplier.run(); + return new RetryResult<>(ret, null, 0); } catch (TException e) { if (reconnect()) { try { - return client.fetchAllConnectionsInfo(); + ret = supplier.run(); + return new RetryResult<>(ret, null, 1); } catch (TException tException) { throw new IoTDBConnectionException(tException); } @@ -1617,6 +1186,12 @@ public TSConnectionInfoResp fetchAllConnections() throws IoTDBConnectionExceptio } } + public TSConnectionInfoResp fetchAllConnections() throws IoTDBConnectionException { + return callWithRetryAndReconnect( + () -> client.fetchAllConnectionsInfo(), resp -> false, resp -> false) + .getResult(); + } + public boolean isEnableRedirect() { return enableRedirect; } @@ -1652,4 +1227,33 @@ private String logForReconnectionFailure() { public String toString() { return "SessionConnection{" + " endPoint=" + endPoint + "}"; } + + private interface TFunction { + T run() throws TException; + } + + private static class RetryResult { + private final T result; + private final TException exception; + private final int retryAttempts; + + public RetryResult(T result, TException exception, int retryAttempts) { + Preconditions.checkArgument(result == null || exception == null); + this.result = result; + this.exception = exception; + this.retryAttempts = retryAttempts; + } + + public int getRetryAttempts() { + return retryAttempts; + } + + public TException getException() { + return exception; + } + + public T getResult() { + return result; + } + } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java new file mode 100644 index 0000000000000..059fdea346948 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.write.record.Tablet; + +public class TableSession implements ITableSession { + + private final Session session; + + TableSession(Session session) { + this.session = session; + } + + @Override + public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException { + session.insertRelationalTablet(tablet); + } + + @Override + public void executeNonQueryStatement(String sql) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement(sql); + } + + @Override + public SessionDataSet executeQueryStatement(String sql) + throws StatementExecutionException, IoTDBConnectionException { + return session.executeQueryStatement(sql); + } + + @Override + public SessionDataSet executeQueryStatement(String sql, long timeoutInMs) + throws StatementExecutionException, IoTDBConnectionException { + return session.executeQueryStatement(sql, timeoutInMs); + } + + @Override + public void close() throws IoTDBConnectionException { + session.close(); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java new file mode 100644 index 0000000000000..fd59846c6e68d --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.rpc.IoTDBConnectionException; + +import java.time.ZoneId; +import java.util.Collections; +import java.util.List; + +import static org.apache.iotdb.session.Session.TABLE; + +/** + * A builder class for constructing instances of {@link ITableSession}. + * + *

This builder provides a fluent API for configuring various options such as connection + * settings, query parameters, and security features. + * + *

All configurations have reasonable default values, which can be overridden as needed. + */ +public class TableSessionBuilder extends AbstractSessionBuilder { + + private boolean enableCompression = false; + private int connectionTimeoutInMs = SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS; + + /** + * Sets the list of node URLs for the IoTDB cluster. + * + * @param nodeUrls a list of node URLs. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue Collection.singletonList("localhost:6667") + */ + public TableSessionBuilder nodeUrls(List nodeUrls) { + this.nodeUrls = nodeUrls; + return this; + } + + /** + * Sets the username for the connection. + * + * @param username the username. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue "root" + */ + public TableSessionBuilder username(String username) { + this.username = username; + return this; + } + + /** + * Sets the password for the connection. + * + * @param password the password. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue "root" + */ + public TableSessionBuilder password(String password) { + this.pw = password; + return this; + } + + /** + * Sets the target database name. + * + * @param database the database name. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue null + */ + public TableSessionBuilder database(String database) { + this.database = database; + return this; + } + + /** + * Sets the query timeout in milliseconds. + * + * @param queryTimeoutInMs the query timeout in milliseconds. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 60000 (1 minute) + */ + public TableSessionBuilder queryTimeoutInMs(long queryTimeoutInMs) { + this.timeOut = queryTimeoutInMs; + return this; + } + + /** + * Sets the fetch size for query results. + * + * @param fetchSize the fetch size. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 5000 + */ + public TableSessionBuilder fetchSize(int fetchSize) { + this.fetchSize = fetchSize; + return this; + } + + /** + * Sets the {@link ZoneId} for timezone-related operations. + * + * @param zoneId the {@link ZoneId}. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue ZoneId.systemDefault() + */ + public TableSessionBuilder zoneId(ZoneId zoneId) { + this.zoneId = zoneId; + return this; + } + + /** + * Sets the default init buffer size for the Thrift client. + * + * @param thriftDefaultBufferSize the buffer size in bytes. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 1024 (1 KB) + */ + public TableSessionBuilder thriftDefaultBufferSize(int thriftDefaultBufferSize) { + this.thriftDefaultBufferSize = thriftDefaultBufferSize; + return this; + } + + /** + * Sets the maximum frame size for the Thrift client. + * + * @param thriftMaxFrameSize the maximum frame size in bytes. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 64 * 1024 * 1024 (64 MB) + */ + public TableSessionBuilder thriftMaxFrameSize(int thriftMaxFrameSize) { + this.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + /** + * Enables or disables redirection for cluster nodes. + * + * @param enableRedirection whether to enable redirection. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue true + */ + public TableSessionBuilder enableRedirection(boolean enableRedirection) { + this.enableRedirection = enableRedirection; + return this; + } + + /** + * Enables or disables automatic fetching of available DataNodes. + * + * @param enableAutoFetch whether to enable automatic fetching. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue true + */ + public TableSessionBuilder enableAutoFetch(boolean enableAutoFetch) { + this.enableAutoFetch = enableAutoFetch; + return this; + } + + /** + * Sets the maximum number of retries for connection attempts. + * + * @param maxRetryCount the maximum retry count. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 60 + */ + public TableSessionBuilder maxRetryCount(int maxRetryCount) { + this.maxRetryCount = maxRetryCount; + return this; + } + + /** + * Sets the interval between retries in milliseconds. + * + * @param retryIntervalInMs the interval in milliseconds. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 500 milliseconds + */ + public TableSessionBuilder retryIntervalInMs(long retryIntervalInMs) { + this.retryIntervalInMs = retryIntervalInMs; + return this; + } + + /** + * Enables or disables SSL for secure connections. + * + * @param useSSL whether to enable SSL. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue false + */ + public TableSessionBuilder useSSL(boolean useSSL) { + this.useSSL = useSSL; + return this; + } + + /** + * Sets the trust store path for SSL connections. + * + * @param keyStore the trust store path. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue null + */ + public TableSessionBuilder trustStore(String keyStore) { + this.trustStore = keyStore; + return this; + } + + /** + * Sets the trust store password for SSL connections. + * + * @param keyStorePwd the trust store password. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue null + */ + public TableSessionBuilder trustStorePwd(String keyStorePwd) { + this.trustStorePwd = keyStorePwd; + return this; + } + + /** + * Enables or disables rpc compression for the connection. + * + * @param enableCompression whether to enable compression. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue false + */ + public TableSessionBuilder enableCompression(boolean enableCompression) { + this.enableCompression = enableCompression; + return this; + } + + /** + * Sets the connection timeout in milliseconds. + * + * @param connectionTimeoutInMs the connection timeout in milliseconds. + * @return the current {@link TableSessionBuilder} instance. + * @defaultValue 0 (no timeout) + */ + public TableSessionBuilder connectionTimeoutInMs(int connectionTimeoutInMs) { + this.connectionTimeoutInMs = connectionTimeoutInMs; + return this; + } + + /** + * Builds and returns a configured {@link ITableSession} instance. + * + * @return a fully configured {@link ITableSession}. + * @throws IoTDBConnectionException if an error occurs while establishing the connection. + */ + public ITableSession build() throws IoTDBConnectionException { + if (nodeUrls == null) { + this.nodeUrls = + Collections.singletonList(SessionConfig.DEFAULT_HOST + ":" + SessionConfig.DEFAULT_PORT); + } + this.sqlDialect = TABLE; + Session newSession = new Session(this); + newSession.open(enableCompression, connectionTimeoutInMs); + return new TableSession(newSession); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java index 65dc1f3ad2cdc..5f9f2a304a550 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java @@ -60,6 +60,8 @@ public class ThriftConnection { protected long statementId; private ZoneId zoneId; + private int timeFactor; + public ThriftConnection( TEndPoint endPoint, int thriftDefaultBufferSize, @@ -124,6 +126,8 @@ public void init( RpcUtils.verifySuccess(openResp.getStatus()); + this.timeFactor = RpcUtils.getTimeFactor(openResp); + if (Session.protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { LOGGER.warn( "Protocol differ, Client version is {}}, but Server version is {}", @@ -175,7 +179,10 @@ protected SessionDataSet executeQueryStatement(String sql, long timeout, int fet timeout, execResp.moreData, fetchSize, - zoneId); + zoneId, + timeFactor, + execResp.isSetTableModel() && execResp.isTableModel(), + execResp.getColumnIndex2TsBlockColumnIndexList()); } public void close() { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/AbstractSessionPoolBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/AbstractSessionPoolBuilder.java new file mode 100644 index 0000000000000..cbadc8cf932c1 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/AbstractSessionPoolBuilder.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.session.AbstractSessionBuilder; + +public class AbstractSessionPoolBuilder extends AbstractSessionBuilder { + int maxSize = SessionConfig.DEFAULT_SESSION_POOL_MAX_SIZE; + long waitToGetSessionTimeoutInMs = 60_000; + boolean enableCompression = false; + int connectionTimeoutInMs = SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS; +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index 70e4d73209df0..3eefa646546a0 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.isession.INodeSupplier; import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.pool.ISessionPool; @@ -40,6 +41,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.record.Tablet; @@ -57,6 +59,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import static org.apache.iotdb.rpc.RpcUtils.isSetSqlDialect; +import static org.apache.iotdb.rpc.RpcUtils.isUseDatabase; + /** * SessionPool is a wrapper of a Session Set. Using SessionPool, the user do not need to consider * how to reuse a session connection. Even if the session is disconnected, the session pool can @@ -124,6 +129,7 @@ public class SessionPool implements ISessionPool { private boolean enableQueryRedirection = false; private Map deviceIdToEndpoint; + private Map tableModelDeviceIdToEndpoint; private int thriftDefaultBufferSize; private int thriftMaxFrameSize; @@ -167,6 +173,11 @@ public class SessionPool implements ISessionPool { protected long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; + protected String sqlDialect = SessionConfig.SQL_DIALECT; + + // may be null + protected String database; + private static final String INSERT_RECORD_FAIL = "insertRecord failed"; private static final String INSERT_RECORD_ERROR_MSG = "unexpected error in insertRecord"; @@ -383,6 +394,7 @@ public SessionPool( this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.connectionTimeoutInMs = connectionTimeoutInMs; this.version = version; @@ -424,6 +436,7 @@ public SessionPool( this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.connectionTimeoutInMs = connectionTimeoutInMs; this.version = version; @@ -468,6 +481,7 @@ public SessionPool( this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.connectionTimeoutInMs = connectionTimeoutInMs; this.version = version; @@ -478,9 +492,9 @@ public SessionPool( initAvailableNodes(SessionUtils.parseSeedNodeUrls(nodeUrls)); } - public SessionPool(Builder builder) { + public SessionPool(AbstractSessionPoolBuilder builder) { this.maxSize = builder.maxSize; - this.user = builder.user; + this.user = builder.username; this.password = builder.pw; this.fetchSize = builder.fetchSize; this.waitToGetSessionTimeoutInMs = builder.waitToGetSessionTimeoutInMs; @@ -489,6 +503,7 @@ public SessionPool(Builder builder) { this.enableRedirection = builder.enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.enableRecordsAutoConvertTablet = builder.enableRecordsAutoConvertTablet; this.connectionTimeoutInMs = builder.connectionTimeoutInMs; @@ -501,7 +516,9 @@ public SessionPool(Builder builder) { this.trustStorePwd = builder.trustStorePwd; this.maxRetryCount = builder.maxRetryCount; this.retryIntervalInMs = builder.retryIntervalInMs; - this.queryTimeoutInMs = builder.queryTimeoutInMs; + this.sqlDialect = builder.sqlDialect; + this.database = builder.database; + this.queryTimeoutInMs = builder.timeOut; if (enableAutoFetch) { initThreadPool(); @@ -522,7 +539,7 @@ public SessionPool(Builder builder) { } else { this.host = builder.host; - this.port = builder.port; + this.port = builder.rpcPort; this.nodeUrls = null; this.formattedNodeUrls = String.format("%s:%s", host, port); if (enableAutoFetch) { @@ -556,6 +573,8 @@ private Session constructNewSession() { .trustStorePwd(trustStorePwd) .maxRetryCount(maxRetryCount) .retryIntervalInMs(retryIntervalInMs) + .sqlDialect(sqlDialect) + .database(database) .timeOut(queryTimeoutInMs) .build(); } else { @@ -577,6 +596,8 @@ private Session constructNewSession() { .trustStorePwd(trustStorePwd) .maxRetryCount(maxRetryCount) .retryIntervalInMs(retryIntervalInMs) + .sqlDialect(sqlDialect) + .database(database) .timeOut(queryTimeoutInMs) .build(); } @@ -651,11 +672,10 @@ private ISession getSession() throws IoTDBConnectionException { long timeOut = Math.min(waitToGetSessionTimeoutInMs, 60_000); if (System.currentTimeMillis() - start > timeOut) { LOGGER.warn( - "the SessionPool has wait for {} seconds to get a new connection: {} with {}, {}", + "the SessionPool has wait for {} seconds to get a new connection: {} with {}", (System.currentTimeMillis() - start) / 1000, formattedNodeUrls, - user, - password); + user); LOGGER.warn( "current occupied size {}, queue size {}, considered size {} ", occupied.size(), @@ -689,7 +709,12 @@ private ISession getSession() throws IoTDBConnectionException { session = constructNewSession(); try { - session.open(enableCompression, connectionTimeoutInMs, deviceIdToEndpoint, availableNodes); + session.open( + enableCompression, + connectionTimeoutInMs, + deviceIdToEndpoint, + tableModelDeviceIdToEndpoint, + availableNodes); // avoid someone has called close() the session pool synchronized (this) { if (closed) { @@ -713,6 +738,10 @@ private ISession getSession() throws IoTDBConnectionException { return session; } + protected ITableSession getPooledTableSession() throws IoTDBConnectionException { + return new TableSessionWrapper((Session) getSession(), this); + } + @Override public int currentAvailableSize() { return queue.size(); @@ -724,7 +753,7 @@ public int currentOccupiedSize() { } @SuppressWarnings({"squid:S2446"}) - private void putBack(ISession session) { + protected void putBack(ISession session) { queue.push(session); synchronized (this) { // we do not need to notifyAll as any waited thread can continue to work after waked up. @@ -794,7 +823,12 @@ public void closeResultSet(SessionDataSetWrapper wrapper) { private void tryConstructNewSession() { Session session = constructNewSession(); try { - session.open(enableCompression, connectionTimeoutInMs, deviceIdToEndpoint, availableNodes); + session.open( + enableCompression, + connectionTimeoutInMs, + deviceIdToEndpoint, + tableModelDeviceIdToEndpoint, + availableNodes); // avoid someone has called close() the session pool synchronized (this) { if (closed) { @@ -838,6 +872,11 @@ private void cleanSessionAndMayThrowConnectionException( } } + protected void cleanSessionAndMayThrowConnectionException(ISession session) { + closeSession(session); + tryConstructNewSession(); + } + /** * insert the data of a device. For each timestamp, the number of measurements is the same. * @@ -3040,6 +3079,13 @@ public SessionDataSetWrapper executeQueryStatement(String sql, long timeoutInMs) @Override public void executeNonQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { + + // 'use XXX' and 'set sql_dialect' is forbidden in SessionPool.executeNonQueryStatement + if (isUseDatabase(sql) || isSetSqlDialect(sql)) { + throw new IllegalArgumentException( + String.format("SessionPool doesn't support executing %s directly", sql)); + } + ISession session = getSession(); try { session.executeNonQueryStatement(sql); @@ -3409,6 +3455,7 @@ public void setEnableRedirection(boolean enableRedirection) { this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } for (ISession session : queue) { session.setEnableRedirection(enableRedirection); @@ -3517,52 +3564,7 @@ public long getQueryTimeout() { return queryTimeoutInMs; } - public static class Builder { - - private String host = SessionConfig.DEFAULT_HOST; - private int port = SessionConfig.DEFAULT_PORT; - private List nodeUrls = null; - private int maxSize = SessionConfig.DEFAULT_SESSION_POOL_MAX_SIZE; - private String user = SessionConfig.DEFAULT_USER; - private String pw = SessionConfig.DEFAULT_PASSWORD; - private int fetchSize = SessionConfig.DEFAULT_FETCH_SIZE; - private long waitToGetSessionTimeoutInMs = 60_000; - private int thriftDefaultBufferSize = SessionConfig.DEFAULT_INITIAL_BUFFER_CAPACITY; - private int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; - private boolean enableCompression = false; - private ZoneId zoneId = null; - - // this field only take effect in write request, nothing to do with any other type requests, - // like query, load and so on. - // if set to true, it means that we may redirect the write request to its corresponding leader - // if set to false, it means that we will only send write request to first available DataNode(it - // may be changed while current DataNode is not available, for example, we may retry to connect - // to another available DataNode) - // so even if enableRedirection is set to false, we may also send write request to another - // datanode while encountering retriable errors in current DataNode - private boolean enableRedirection = SessionConfig.DEFAULT_REDIRECTION_MODE; - private boolean enableRecordsAutoConvertTablet = - SessionConfig.DEFAULT_RECORDS_AUTO_CONVERT_TABLET; - private int connectionTimeoutInMs = SessionConfig.DEFAULT_CONNECTION_TIMEOUT_MS; - private Version version = SessionConfig.DEFAULT_VERSION; - - private boolean useSSL = false; - private String trustStore; - private String trustStorePwd; - - // set to true, means that we will start a background thread to fetch all available (Status is - // not Removing) datanodes in cluster, and these available nodes will be used in retrying stage - private boolean enableAutoFetch; - - // max retry count, if set to 0, means that we won't do any retry - // we can use any available DataNodes(fetched in background thread if enableAutoFetch is true, - // or nodeUrls user specified) to retry, even if enableRedirection is false - private int maxRetryCount = SessionConfig.MAX_RETRY_COUNT; - - // sleep time between each retry - private long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; - - private long queryTimeoutInMs = SessionConfig.DEFAULT_QUERY_TIME_OUT; + public static class Builder extends AbstractSessionPoolBuilder { public Builder useSSL(boolean useSSL) { this.useSSL = useSSL; @@ -3585,7 +3587,7 @@ public Builder host(String host) { } public Builder port(int port) { - this.port = port; + this.rpcPort = port; return this; } @@ -3600,7 +3602,7 @@ public Builder maxSize(int maxSize) { } public Builder user(String user) { - this.user = user; + this.username = user; return this; } @@ -3675,7 +3677,7 @@ public Builder retryIntervalInMs(long retryIntervalInMs) { } public Builder queryTimeoutInMs(long queryTimeoutInMs) { - this.queryTimeoutInMs = queryTimeoutInMs; + this.timeOut = queryTimeoutInMs; return this; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionPool.java new file mode 100644 index 0000000000000..4e08f202a14ad --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionPool.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; + +public class TableSessionPool implements ITableSessionPool { + + private final SessionPool sessionPool; + + TableSessionPool(SessionPool sessionPool) { + this.sessionPool = sessionPool; + } + + @Override + public ITableSession getSession() throws IoTDBConnectionException { + return sessionPool.getPooledTableSession(); + } + + @Override + public void close() { + this.sessionPool.close(); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionPoolBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionPoolBuilder.java new file mode 100644 index 0000000000000..4deb94239463d --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionPoolBuilder.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import java.time.ZoneId; +import java.util.Collections; +import java.util.List; + +/** + * A builder class for constructing instances of {@link ITableSessionPool}. + * + *

This builder provides a fluent API for configuring a session pool, including connection + * settings, session parameters, and pool behavior. + * + *

All configurations have reasonable default values, which can be overridden as needed. + */ +public class TableSessionPoolBuilder extends AbstractSessionPoolBuilder { + + /** + * Sets the list of node URLs for the IoTDB cluster. + * + * @param nodeUrls a list of node URLs. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue Collection.singletonList("localhost:6667") + */ + public TableSessionPoolBuilder nodeUrls(List nodeUrls) { + this.nodeUrls = nodeUrls; + return this; + } + + /** + * Sets the maximum size of the session pool. + * + * @param maxSize the maximum number of sessions allowed in the pool. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 5 + */ + public TableSessionPoolBuilder maxSize(int maxSize) { + this.maxSize = maxSize; + return this; + } + + /** + * Sets the username for the connection. + * + * @param user the username. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue "root" + */ + public TableSessionPoolBuilder user(String user) { + this.username = user; + return this; + } + + /** + * Sets the password for the connection. + * + * @param password the password. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue "root" + */ + public TableSessionPoolBuilder password(String password) { + this.pw = password; + return this; + } + + /** + * Sets the target database name. + * + * @param database the database name. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue null + */ + public TableSessionPoolBuilder database(String database) { + this.database = database; + return this; + } + + /** + * Sets the query timeout in milliseconds. + * + * @param queryTimeoutInMs the query timeout in milliseconds. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 60000 (1 minute) + */ + public TableSessionPoolBuilder queryTimeoutInMs(long queryTimeoutInMs) { + this.timeOut = queryTimeoutInMs; + return this; + } + + /** + * Sets the fetch size for query results. + * + * @param fetchSize the fetch size. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 5000 + */ + public TableSessionPoolBuilder fetchSize(int fetchSize) { + this.fetchSize = fetchSize; + return this; + } + + /** + * Sets the {@link ZoneId} for timezone-related operations. + * + * @param zoneId the {@link ZoneId}. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue ZoneId.systemDefault() + */ + public TableSessionPoolBuilder zoneId(ZoneId zoneId) { + this.zoneId = zoneId; + return this; + } + + /** + * Sets the timeout for waiting to acquire a session from the pool. + * + * @param waitToGetSessionTimeoutInMs the timeout duration in milliseconds. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 60000 (60 seconds) + */ + public TableSessionPoolBuilder waitToGetSessionTimeoutInMs(long waitToGetSessionTimeoutInMs) { + this.waitToGetSessionTimeoutInMs = waitToGetSessionTimeoutInMs; + return this; + } + + /** + * Sets the default buffer size for the Thrift client. + * + * @param thriftDefaultBufferSize the buffer size in bytes. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 1024 (1 KB) + */ + public TableSessionPoolBuilder thriftDefaultBufferSize(int thriftDefaultBufferSize) { + this.thriftDefaultBufferSize = thriftDefaultBufferSize; + return this; + } + + /** + * Sets the maximum frame size for the Thrift client. + * + * @param thriftMaxFrameSize the maximum frame size in bytes. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 64 * 1024 * 1024 (64 MB) + */ + public TableSessionPoolBuilder thriftMaxFrameSize(int thriftMaxFrameSize) { + this.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + /** + * Enables or disables rpc compression for the connection. + * + * @param enableCompression whether to enable compression. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue false + */ + public TableSessionPoolBuilder enableCompression(boolean enableCompression) { + this.enableCompression = enableCompression; + return this; + } + + /** + * Enables or disables redirection for cluster nodes. + * + * @param enableRedirection whether to enable redirection. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue true + */ + public TableSessionPoolBuilder enableRedirection(boolean enableRedirection) { + this.enableRedirection = enableRedirection; + return this; + } + + /** + * Sets the connection timeout in milliseconds. + * + * @param connectionTimeoutInMs the connection timeout in milliseconds. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 0 (no timeout) + */ + public TableSessionPoolBuilder connectionTimeoutInMs(int connectionTimeoutInMs) { + this.connectionTimeoutInMs = connectionTimeoutInMs; + return this; + } + + /** + * Enables or disables automatic fetching of available DataNodes. + * + * @param enableAutoFetch whether to enable automatic fetching. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue true + */ + public TableSessionPoolBuilder enableAutoFetch(boolean enableAutoFetch) { + this.enableAutoFetch = enableAutoFetch; + return this; + } + + /** + * Sets the maximum number of retries for connection attempts. + * + * @param maxRetryCount the maximum retry count. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 60 + */ + public TableSessionPoolBuilder maxRetryCount(int maxRetryCount) { + this.maxRetryCount = maxRetryCount; + return this; + } + + /** + * Sets the interval between retries in milliseconds. + * + * @param retryIntervalInMs the interval in milliseconds. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue 500 milliseconds + */ + public TableSessionPoolBuilder retryIntervalInMs(long retryIntervalInMs) { + this.retryIntervalInMs = retryIntervalInMs; + return this; + } + + /** + * Enables or disables SSL for secure connections. + * + * @param useSSL whether to enable SSL. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue false + */ + public TableSessionPoolBuilder useSSL(boolean useSSL) { + this.useSSL = useSSL; + return this; + } + + /** + * Sets the trust store path for SSL connections. + * + * @param keyStore the trust store path. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue null + */ + public TableSessionPoolBuilder trustStore(String keyStore) { + this.trustStore = keyStore; + return this; + } + + /** + * Sets the trust store password for SSL connections. + * + * @param keyStorePwd the trust store password. + * @return the current {@link TableSessionPoolBuilder} instance. + * @defaultValue null + */ + public TableSessionPoolBuilder trustStorePwd(String keyStorePwd) { + this.trustStorePwd = keyStorePwd; + return this; + } + + /** + * Builds and returns a configured {@link ITableSessionPool} instance. + * + * @return a fully configured {@link ITableSessionPool}. + */ + public ITableSessionPool build() { + if (nodeUrls == null) { + this.nodeUrls = + Collections.singletonList(SessionConfig.DEFAULT_HOST + ":" + SessionConfig.DEFAULT_PORT); + } + this.sqlDialect = "table"; + SessionPool sessionPool = new SessionPool(this); + return new TableSessionPool(sessionPool); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java new file mode 100644 index 0000000000000..761bb3a40d3b1 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * NOTICE: TableSessionWrapper is specific to the table model. + * + *

used for SessionPool.getSession need to do some other things like calling + * cleanSessionAndMayThrowConnectionException in SessionPool while encountering connection exception + * only need to putBack to SessionPool while closing. + */ +public class TableSessionWrapper implements ITableSession { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableSessionWrapper.class); + + private Session session; + + private final SessionPool sessionPool; + + private final AtomicBoolean closed; + + protected TableSessionWrapper(Session session, SessionPool sessionPool) { + this.session = session; + this.sessionPool = sessionPool; + this.closed = new AtomicBoolean(false); + } + + @Override + public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException { + try { + session.insertRelationalTablet(tablet); + } catch (IoTDBConnectionException e) { + sessionPool.cleanSessionAndMayThrowConnectionException(session); + closed.set(true); + session = null; + throw e; + } + } + + @Override + public SessionDataSet executeQueryStatement(String sql) + throws StatementExecutionException, IoTDBConnectionException { + try { + return session.executeQueryStatement(sql); + } catch (IoTDBConnectionException e) { + sessionPool.cleanSessionAndMayThrowConnectionException(session); + closed.set(true); + session = null; + throw e; + } + } + + @Override + public SessionDataSet executeQueryStatement(String sql, long timeoutInMs) + throws StatementExecutionException, IoTDBConnectionException { + try { + return session.executeQueryStatement(sql, timeoutInMs); + } catch (IoTDBConnectionException e) { + sessionPool.cleanSessionAndMayThrowConnectionException(session); + closed.set(true); + session = null; + throw e; + } + } + + @Override + public void executeNonQueryStatement(String sql) + throws IoTDBConnectionException, StatementExecutionException { + try { + session.executeNonQueryStatement(sql); + } catch (IoTDBConnectionException e) { + sessionPool.cleanSessionAndMayThrowConnectionException(session); + closed.set(true); + session = null; + throw e; + } + } + + @Override + public void close() throws IoTDBConnectionException { + if (!Objects.equals(session.getSqlDialect(), sessionPool.sqlDialect)) { + try { + session.executeNonQueryStatement("set sql_dialect=" + sessionPool.sqlDialect); + } catch (StatementExecutionException e) { + LOGGER.warn( + "Failed to change back sql_dialect by executing: set sql_dialect={}", + sessionPool.sqlDialect, + e); + session.close(); + session = null; + return; + } + } + + if (closed.compareAndSet(false, true)) { + if (!Objects.equals(session.getDatabase(), sessionPool.database) + && sessionPool.database != null) { + try { + session.executeNonQueryStatement("use " + sessionPool.database); + } catch (StatementExecutionException e) { + LOGGER.warn( + "Failed to change back database by executing: use {}", sessionPool.database, e); + session.close(); + session = null; + return; + } + } + + sessionPool.putBack(session); + session = null; + } + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java new file mode 100644 index 0000000000000..5ab42f29d70dd --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/AbstractSubscriptionSession.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.model.Subscription; +import org.apache.iotdb.session.subscription.model.Topic; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; + +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +abstract class AbstractSubscriptionSession { + + private final SubscriptionSessionWrapper session; + + protected AbstractSubscriptionSession(final SubscriptionSessionWrapper session) { + this.session = session; + } + + public SubscriptionSessionConnection getSessionConnection() { + return session.getSessionConnection(); + } + + public int getThriftMaxFrameSize() { + return session.getThriftMaxFrameSize(); + } + + /////////////////////////////// open & close /////////////////////////////// + + protected void open() throws IoTDBConnectionException { + session.open(); + } + + protected void close() throws IoTDBConnectionException { + session.close(); + } + + /////////////////////////////// topic /////////////////////////////// + + protected void createTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + final String sql = String.format("CREATE TOPIC %s", topicName); + session.executeNonQueryStatement(sql); + } + + protected void createTopicIfNotExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + final String sql = String.format("CREATE TOPIC IF NOT EXISTS %s", topicName); + session.executeNonQueryStatement(sql); + } + + protected void createTopic(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + createTopic(topicName, properties, false); + } + + protected void createTopicIfNotExists(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + createTopic(topicName, properties, true); + } + + private void createTopic( + final String topicName, final Properties properties, final boolean isSetIfNotExistsCondition) + throws IoTDBConnectionException, StatementExecutionException { + if (Objects.isNull(properties) || properties.isEmpty()) { + if (isSetIfNotExistsCondition) { + createTopicIfNotExists(topicName); + } else { + createTopic(topicName); + } + return; + } + final StringBuilder sb = new StringBuilder(); + sb.append('('); + properties.forEach( + (k, v) -> + sb.append('\'') + .append(k) + .append('\'') + .append('=') + .append('\'') + .append(v) + .append('\'') + .append(',')); + sb.deleteCharAt(sb.length() - 1); + sb.append(')'); + final String sql = + isSetIfNotExistsCondition + ? String.format("CREATE TOPIC IF NOT EXISTS %s WITH %s", topicName, sb) + : String.format("CREATE TOPIC %s WITH %s", topicName, sb); + session.executeNonQueryStatement(sql); + } + + protected void dropTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + final String sql = String.format("DROP TOPIC %s", topicName); + session.executeNonQueryStatement(sql); + } + + protected void dropTopicIfExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + final String sql = String.format("DROP TOPIC IF EXISTS %s", topicName); + session.executeNonQueryStatement(sql); + } + + protected Set getTopics() throws IoTDBConnectionException, StatementExecutionException { + final String sql = "SHOW TOPICS"; + try (final SessionDataSet dataSet = session.executeQueryStatement(sql)) { + return convertDataSetToTopics(dataSet); + } + } + + protected Optional getTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + final String sql = String.format("SHOW TOPIC %s", topicName); + try (final SessionDataSet dataSet = session.executeQueryStatement(sql)) { + final Set topics = convertDataSetToTopics(dataSet); + if (topics.isEmpty()) { + return Optional.empty(); + } + return Optional.of(topics.iterator().next()); + } + } + + /////////////////////////////// subscription /////////////////////////////// + + protected Set getSubscriptions() + throws IoTDBConnectionException, StatementExecutionException { + final String sql = "SHOW SUBSCRIPTIONS"; + try (final SessionDataSet dataSet = session.executeQueryStatement(sql)) { + return convertDataSetToSubscriptions(dataSet); + } + } + + protected Set getSubscriptions(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(topicName); // ignore the parse result + final String sql = String.format("SHOW SUBSCRIPTIONS ON %s", topicName); + try (final SessionDataSet dataSet = session.executeQueryStatement(sql)) { + return convertDataSetToSubscriptions(dataSet); + } + } + + protected void dropSubscription(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(subscriptionId); // ignore the parse result + final String sql = String.format("DROP SUBSCRIPTION %s", subscriptionId); + session.executeNonQueryStatement(sql); + } + + protected void dropSubscriptionIfExists(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException { + IdentifierUtils.checkAndParseIdentifier(subscriptionId); // ignore the parse result + final String sql = String.format("DROP SUBSCRIPTION IF EXISTS %s", subscriptionId); + session.executeNonQueryStatement(sql); + } + + /////////////////////////////// utility /////////////////////////////// + + private Set convertDataSetToTopics(final SessionDataSet dataSet) + throws IoTDBConnectionException, StatementExecutionException { + final Set topics = new HashSet<>(); + while (dataSet.hasNext()) { + final RowRecord record = dataSet.next(); + final List fields = record.getFields(); + if (fields.size() != 2) { + throw new SubscriptionException( + String.format( + "Unexpected fields %s was obtained during SHOW TOPIC...", + fields.stream().map(Object::toString).collect(Collectors.joining(", ")))); + } + topics.add(new Topic(fields.get(0).getStringValue(), fields.get(1).getStringValue())); + } + return topics; + } + + private Set convertDataSetToSubscriptions(final SessionDataSet dataSet) + throws IoTDBConnectionException, StatementExecutionException { + final Set subscriptions = new HashSet<>(); + while (dataSet.hasNext()) { + final RowRecord record = dataSet.next(); + final List fields = record.getFields(); + if (fields.size() != 4) { + throw new SubscriptionException( + String.format( + "Unexpected fields %s was obtained during SHOW SUBSCRIPTION...", + fields.stream().map(Object::toString).collect(Collectors.joining(", ")))); + } + subscriptions.add( + new Subscription( + fields.get(0).getStringValue(), + fields.get(1).getStringValue(), + fields.get(2).getStringValue(), + fields.get(3).getStringValue())); + } + return subscriptions; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ISubscriptionTableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ISubscriptionTableSession.java new file mode 100644 index 0000000000000..2dac0f5106b96 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ISubscriptionTableSession.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.model.Subscription; +import org.apache.iotdb.session.subscription.model.Topic; + +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +/** + * This interface defines methods for managing topics and subscriptions of table model in an IoTDB + * environment. + */ +public interface ISubscriptionTableSession extends AutoCloseable { + + /** + * Opens this session and establishes a connection to IoTDB. + * + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + */ + void open() throws IoTDBConnectionException; + + /////////////////////////////// topic /////////////////////////////// + + /** + * Creates a topic with the specified name. + * + *

If the topic name contains single quotes, it must be enclosed in backticks (`). For example, + * to create a topic named 'topic', the value passed in as topicName should be `'topic'`. + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void createTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Creates a topic with the specified name only if it does not already exist. + * + *

This method is similar to {@link #createTopic(String)}, but includes the 'IF NOT EXISTS' + * condition. If the topic name contains single quotes, it must be enclosed in backticks (`). + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void createTopicIfNotExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Creates a topic with the specified name and properties. + * + *

Topic names with single quotes must be enclosed in backticks (`). Property keys and values + * are included in the SQL statement automatically. + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @param properties A {@link Properties} object containing the topic's properties. + * @throws IoTDBConnectionException If a connection issue occurs with IoTDB. + * @throws StatementExecutionException If a statement execution issue occurs. + */ + void createTopic(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Creates a topic with the specified properties if it does not already exist. Topic names with + * single quotes must be enclosed in backticks (`). + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @param properties A {@link Properties} object containing the topic's properties. + * @throws IoTDBConnectionException If a connection issue occurs. + * @throws StatementExecutionException If the SQL statement execution fails. + */ + void createTopicIfNotExists(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Drops the specified topic. + * + *

This method removes the specified topic from the database. If the topic name contains single + * quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic to be deleted. If it contains single quotes, it needs to + * be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Drops the specified topic if it exists. + * + *

This method is similar to {@link #dropTopic(String)}, but includes the 'IF EXISTS' + * condition. If the topic name contains single quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic to be deleted. If it contains single quotes, it needs to + * be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropTopicIfExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Retrieves all existing topics from the IoTDB environment. + * + * @return A set of {@link Topic} objects representing all existing topics. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Set getTopics() throws IoTDBConnectionException, StatementExecutionException; + + /** + * Retrieves an optional topic by its name. If the topic does not exist, an empty {@link Optional} + * is returned. + * + *

If the topic name contains single quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic to retrieve. If it contains single quotes, it needs to + * be enclosed in backticks. + * @return An {@link Optional} containing the {@link Topic}, or empty if the topic is not found. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Optional getTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /////////////////////////////// subscription /////////////////////////////// + + /** + * Retrieves all existing subscriptions from the IoTDB environment. + * + * @return A set of {@link Subscription} objects representing all existing subscriptions. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Set getSubscriptions() throws IoTDBConnectionException, StatementExecutionException; + + /** + * Retrieves all subscriptions belonging to a specific topic. If the topic name contains single + * quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic whose subscriptions are to be retrieved. If it contains + * single quotes, it needs to be enclosed in backticks. + * @return A set of {@link Subscription} objects for the specified topic. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Set getSubscriptions(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Removes the subscription identified by the given subscription ID. + * + * @param subscriptionId The unique identifier of the subscription to be removed. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropSubscription(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Removes the subscription identified by the given subscription ID if it exists. + * + *

If the subscription does not exist, this method will not throw an exception. + * + * @param subscriptionId The unique identifier of the subscription to be removed. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropSubscriptionIfExists(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException; +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ISubscriptionTreeSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ISubscriptionTreeSession.java new file mode 100644 index 0000000000000..799314769e8eb --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ISubscriptionTreeSession.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.model.Subscription; +import org.apache.iotdb.session.subscription.model.Topic; + +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +/** + * This interface defines methods for managing topics and subscriptions of tree model in an IoTDB + * environment. + */ +public interface ISubscriptionTreeSession extends AutoCloseable { + + /** + * Opens this session and establishes a connection to IoTDB. + * + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + */ + void open() throws IoTDBConnectionException; + + /////////////////////////////// topic /////////////////////////////// + + /** + * Creates a topic with the specified name. + * + *

If the topic name contains single quotes, it must be enclosed in backticks (`). For example, + * to create a topic named 'topic', the value passed in as topicName should be `'topic'`. + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void createTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Creates a topic with the specified name only if it does not already exist. + * + *

This method is similar to {@link #createTopic(String)}, but includes the 'IF NOT EXISTS' + * condition. If the topic name contains single quotes, it must be enclosed in backticks (`). + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void createTopicIfNotExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Creates a topic with the specified name and properties. + * + *

Topic names with single quotes must be enclosed in backticks (`). Property keys and values + * are included in the SQL statement automatically. + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @param properties A {@link Properties} object containing the topic's properties. + * @throws IoTDBConnectionException If a connection issue occurs with IoTDB. + * @throws StatementExecutionException If a statement execution issue occurs. + */ + void createTopic(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Creates a topic with the specified properties if it does not already exist. Topic names with + * single quotes must be enclosed in backticks (`). + * + * @param topicName If the created topic name contains single quotes, the passed parameter needs + * to be enclosed in backticks. + * @param properties A {@link Properties} object containing the topic's properties. + * @throws IoTDBConnectionException If a connection issue occurs. + * @throws StatementExecutionException If the SQL statement execution fails. + */ + void createTopicIfNotExists(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Drops the specified topic. + * + *

This method removes the specified topic from the database. If the topic name contains single + * quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic to be deleted. If it contains single quotes, it needs to + * be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Drops the specified topic if it exists. + * + *

This method is similar to {@link #dropTopic(String)}, but includes the 'IF EXISTS' + * condition. If the topic name contains single quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic to be deleted. If it contains single quotes, it needs to + * be enclosed in backticks. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropTopicIfExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Retrieves all existing topics from the IoTDB environment. + * + * @return A set of {@link Topic} objects representing all existing topics. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Set getTopics() throws IoTDBConnectionException, StatementExecutionException; + + /** + * Retrieves an optional topic by its name. If the topic does not exist, an empty {@link Optional} + * is returned. + * + *

If the topic name contains single quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic to retrieve. If it contains single quotes, it needs to + * be enclosed in backticks. + * @return An {@link Optional} containing the {@link Topic}, or empty if the topic is not found. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Optional getTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /////////////////////////////// subscription /////////////////////////////// + + /** + * Retrieves all existing subscriptions from the IoTDB environment. + * + * @return A set of {@link Subscription} objects representing all existing subscriptions. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Set getSubscriptions() throws IoTDBConnectionException, StatementExecutionException; + + /** + * Retrieves all subscriptions belonging to a specific topic. If the topic name contains single + * quotes, it must be enclosed in backticks (`). + * + * @param topicName The name of the topic whose subscriptions are to be retrieved. If it contains + * single quotes, it needs to be enclosed in backticks. + * @return A set of {@link Subscription} objects for the specified topic. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + Set getSubscriptions(final String topicName) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Removes the subscription identified by the given subscription ID. + * + * @param subscriptionId The unique identifier of the subscription to be removed. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropSubscription(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException; + + /** + * Removes the subscription identified by the given subscription ID if it exists. + * + *

If the subscription does not exist, this method will not throw an exception. + * + * @param subscriptionId The unique identifier of the subscription to be removed. + * @throws IoTDBConnectionException If there is an issue with the connection to IoTDB. + * @throws StatementExecutionException If there is an issue executing the SQL statement. + */ + void dropSubscriptionIfExists(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException; +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSession.java deleted file mode 100644 index d70de467db066..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSession.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.isession.SessionConfig; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionParameterNotValidException; -import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.SessionConnection; -import org.apache.iotdb.session.subscription.model.Subscription; -import org.apache.iotdb.session.subscription.model.Topic; - -import org.apache.tsfile.read.common.Field; -import org.apache.tsfile.read.common.RowRecord; - -import java.time.ZoneId; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; - -public class SubscriptionSession extends Session { - - public SubscriptionSession(final String host, final int port) { - this(host, port, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); - } - - public SubscriptionSession( - final String host, final int port, final String username, final String password) { - // TODO: more configs control - super( - new Session.Builder() - .host(host) - .port(port) - .username(username) - .password(password) - // disable auto fetch - .enableAutoFetch(false) - // disable redirection - .enableRedirection(false) - // TODO: config - .thriftMaxFrameSize(Integer.MAX_VALUE)); - } - - @Override - public SessionConnection constructSessionConnection( - final Session session, final TEndPoint endpoint, final ZoneId zoneId) - throws IoTDBConnectionException { - if (Objects.isNull(endpoint)) { - throw new SubscriptionParameterNotValidException( - "Subscription session must be configured with an endpoint."); - } - return new SubscriptionSessionConnection( - session, endpoint, zoneId, availableNodes, maxRetryCount, retryIntervalInMs); - } - - /////////////////////////////// topic /////////////////////////////// - - public void createTopic(final String topicName) - throws IoTDBConnectionException, StatementExecutionException { - final String sql = String.format("CREATE TOPIC %s", topicName); - executeNonQueryStatement(sql); - } - - public void createTopic(final String topicName, final Properties properties) - throws IoTDBConnectionException, StatementExecutionException { - if (properties.isEmpty()) { - createTopic(topicName); - return; - } - final StringBuilder sb = new StringBuilder(); - sb.append('('); - properties.forEach( - (k, v) -> - sb.append('\'') - .append(k) - .append('\'') - .append('=') - .append('\'') - .append(v) - .append('\'') - .append(',')); - sb.deleteCharAt(sb.length() - 1); - sb.append(')'); - final String sql = String.format("CREATE TOPIC %s WITH %s", topicName, sb); - executeNonQueryStatement(sql); - } - - public void dropTopic(final String topicName) - throws IoTDBConnectionException, StatementExecutionException { - final String sql = String.format("DROP TOPIC %s", topicName); - executeNonQueryStatement(sql); - } - - public Set getTopics() throws IoTDBConnectionException, StatementExecutionException { - final String sql = "SHOW TOPICS"; - try (final SessionDataSet dataSet = executeQueryStatement(sql)) { - return convertDataSetToTopics(dataSet); - } - } - - public Optional getTopic(final String topicName) - throws IoTDBConnectionException, StatementExecutionException { - final String sql = String.format("SHOW TOPIC %s", topicName); - try (final SessionDataSet dataSet = executeQueryStatement(sql)) { - final Set topics = convertDataSetToTopics(dataSet); - if (topics.isEmpty()) { - return Optional.empty(); - } - return Optional.of(topics.iterator().next()); - } - } - - /////////////////////////////// subscription /////////////////////////////// - - public Set getSubscriptions() - throws IoTDBConnectionException, StatementExecutionException { - final String sql = "SHOW SUBSCRIPTIONS"; - try (final SessionDataSet dataSet = executeQueryStatement(sql)) { - return convertDataSetToSubscriptions(dataSet); - } - } - - public Set getSubscriptions(final String topicName) - throws IoTDBConnectionException, StatementExecutionException { - final String sql = String.format("SHOW SUBSCRIPTIONS ON %s", topicName); - try (final SessionDataSet dataSet = executeQueryStatement(sql)) { - return convertDataSetToSubscriptions(dataSet); - } - } - - /////////////////////////////// utility /////////////////////////////// - - public Set convertDataSetToTopics(final SessionDataSet dataSet) - throws IoTDBConnectionException, StatementExecutionException { - final Set topics = new HashSet<>(); - while (dataSet.hasNext()) { - final RowRecord record = dataSet.next(); - final List fields = record.getFields(); - if (fields.size() != 2) { - throw new SubscriptionException( - String.format( - "Unexpected fields %s was obtained during SHOW TOPIC...", - fields.stream().map(Object::toString).collect(Collectors.joining(", ")))); - } - topics.add(new Topic(fields.get(0).getStringValue(), fields.get(1).getStringValue())); - } - return topics; - } - - public Set convertDataSetToSubscriptions(final SessionDataSet dataSet) - throws IoTDBConnectionException, StatementExecutionException { - final Set subscriptions = new HashSet<>(); - while (dataSet.hasNext()) { - final RowRecord record = dataSet.next(); - final List fields = record.getFields(); - if (fields.size() != 3) { - throw new SubscriptionException( - String.format( - "Unexpected fields %s was obtained during SHOW SUBSCRIPTION...", - fields.stream().map(Object::toString).collect(Collectors.joining(", ")))); - } - subscriptions.add( - new Subscription( - fields.get(0).getStringValue(), - fields.get(1).getStringValue(), - fields.get(2).getStringValue())); - } - return subscriptions; - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionConnection.java index 135e1055f47c0..3f15f4dcf6aec 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionConnection.java @@ -20,9 +20,7 @@ package org.apache.iotdb.session.subscription; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeReq; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; import org.apache.iotdb.session.Session; @@ -31,50 +29,30 @@ import org.apache.thrift.TException; import java.time.ZoneId; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Supplier; public class SubscriptionSessionConnection extends SessionConnection { - private static final String SHOW_DATA_NODES_COMMAND = "SHOW DATANODES"; - private static final String NODE_ID_COLUMN_NAME = "NodeID"; - private static final String STATUS_COLUMN_NAME = "Status"; - private static final String IP_COLUMN_NAME = "RpcAddress"; - private static final String PORT_COLUMN_NAME = "RpcPort"; - private static final String REMOVING_STATUS = "Removing"; - public SubscriptionSessionConnection( - Session session, - TEndPoint endPoint, - ZoneId zoneId, - Supplier> availableNodes, - int maxRetryCount, - long retryIntervalInMs) + final Session session, + final TEndPoint endPoint, + final ZoneId zoneId, + final Supplier> availableNodes, + final int maxRetryCount, + final long retryIntervalInMs, + final String sqlDialect, + final String database) throws IoTDBConnectionException { - super(session, endPoint, zoneId, availableNodes, maxRetryCount, retryIntervalInMs); - } - - // from org.apache.iotdb.session.NodesSupplier.updateDataNodeList - public Map fetchAllEndPoints() - throws IoTDBConnectionException, StatementExecutionException { - SessionDataSet dataSet = session.executeQueryStatement(SHOW_DATA_NODES_COMMAND); - SessionDataSet.DataIterator iterator = dataSet.iterator(); - Map endPoints = new HashMap<>(); - while (iterator.next()) { - // ignore removing DN - if (REMOVING_STATUS.equals(iterator.getString(STATUS_COLUMN_NAME))) { - continue; - } - String ip = iterator.getString(IP_COLUMN_NAME); - String port = iterator.getString(PORT_COLUMN_NAME); - if (ip != null && port != null) { - endPoints.put( - iterator.getInt(NODE_ID_COLUMN_NAME), new TEndPoint(ip, Integer.parseInt(port))); - } - } - return endPoints; + super( + session, + endPoint, + zoneId, + availableNodes, + maxRetryCount, + retryIntervalInMs, + sqlDialect, + database); } public TPipeSubscribeResp pipeSubscribe(final TPipeSubscribeReq req) throws TException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionWrapper.java new file mode 100644 index 0000000000000..455e9082ee6f2 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionSessionWrapper.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionParameterNotValidException; +import org.apache.iotdb.session.AbstractSessionBuilder; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.SessionConnection; + +import java.time.ZoneId; +import java.util.Objects; + +public final class SubscriptionSessionWrapper extends Session { + + public SubscriptionSessionWrapper(final AbstractSessionBuilder builder) { + super(builder); + } + + public void open() throws IoTDBConnectionException { + super.open(); + } + + public void close() throws IoTDBConnectionException { + super.close(); + } + + @Override + public SessionConnection constructSessionConnection( + final Session session, final TEndPoint endpoint, final ZoneId zoneId) + throws IoTDBConnectionException { + if (Objects.isNull(endpoint)) { + throw new SubscriptionParameterNotValidException( + "Subscription session must be configured with an endpoint."); + } + return new SubscriptionSessionConnection( + session, + endpoint, + zoneId, + availableNodes, + maxRetryCount, + retryIntervalInMs, + sqlDialect, + database); + } + + public SubscriptionSessionConnection getSessionConnection() { + return (SubscriptionSessionConnection) defaultSessionConnection; + } + + public int getThriftMaxFrameSize() { + return thriftMaxFrameSize; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTableSession.java new file mode 100644 index 0000000000000..bf21a43b26f81 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTableSession.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.AbstractSessionBuilder; +import org.apache.iotdb.session.subscription.model.Subscription; +import org.apache.iotdb.session.subscription.model.Topic; + +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +public class SubscriptionTableSession extends AbstractSubscriptionSession + implements ISubscriptionTableSession { + + public SubscriptionTableSession(final AbstractSessionBuilder builder) { + super(new SubscriptionSessionWrapper(builder)); + } + + /////////////////////////////// open & close /////////////////////////////// + + @Override + public void open() throws IoTDBConnectionException { + super.open(); + } + + @Override + public void close() throws IoTDBConnectionException { + super.close(); + } + + /////////////////////////////// topic /////////////////////////////// + + @Override + public void createTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopic(topicName); + } + + @Override + public void createTopicIfNotExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopicIfNotExists(topicName); + } + + @Override + public void createTopic(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopic(topicName, properties); + } + + @Override + public void createTopicIfNotExists(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopicIfNotExists(topicName, properties); + } + + @Override + public void dropTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.dropTopic(topicName); + } + + @Override + public void dropTopicIfExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.dropTopicIfExists(topicName); + } + + @Override + public Set getTopics() throws IoTDBConnectionException, StatementExecutionException { + return super.getTopics(); + } + + @Override + public Optional getTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + return super.getTopic(topicName); + } + + /////////////////////////////// subscription /////////////////////////////// + + @Override + public Set getSubscriptions() + throws IoTDBConnectionException, StatementExecutionException { + return super.getSubscriptions(); + } + + @Override + public Set getSubscriptions(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + return super.getSubscriptions(topicName); + } + + @Override + public void dropSubscription(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException { + super.dropSubscription(subscriptionId); + } + + @Override + public void dropSubscriptionIfExists(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException { + super.dropSubscriptionIfExists(subscriptionId); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTableSessionBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTableSessionBuilder.java new file mode 100644 index 0000000000000..41e26161c04c6 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTableSessionBuilder.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.session.AbstractSessionBuilder; + +public class SubscriptionTableSessionBuilder extends AbstractSessionBuilder { + + public SubscriptionTableSessionBuilder() { + // use table model + super.sqlDialect = "table"; + // disable auto fetch + super.enableAutoFetch = false; + // disable redirection + super.enableRedirection = false; + } + + public SubscriptionTableSessionBuilder host(final String host) { + super.host = host; + return this; + } + + public SubscriptionTableSessionBuilder port(final int port) { + super.rpcPort = port; + return this; + } + + public SubscriptionTableSessionBuilder username(final String username) { + super.username = username; + return this; + } + + public SubscriptionTableSessionBuilder password(final String password) { + super.pw = password; + return this; + } + + public SubscriptionTableSessionBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + public ISubscriptionTableSession build() throws IoTDBConnectionException { + final ISubscriptionTableSession session = new SubscriptionTableSession(this); + session.open(); + return session; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTreeSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTreeSession.java new file mode 100644 index 0000000000000..900a243434d1c --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTreeSession.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.AbstractSessionBuilder; +import org.apache.iotdb.session.subscription.model.Subscription; +import org.apache.iotdb.session.subscription.model.Topic; + +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +public class SubscriptionTreeSession extends AbstractSubscriptionSession + implements ISubscriptionTreeSession { + + public SubscriptionTreeSession(final AbstractSessionBuilder builder) { + super(new SubscriptionSessionWrapper(builder)); + } + + @Deprecated // keep for forward compatibility + public SubscriptionTreeSession(final String host, final int port) { + super( + new SubscriptionSessionWrapper(new SubscriptionTreeSessionBuilder().host(host).port(port))); + } + + @Deprecated // keep for forward compatibility + public SubscriptionTreeSession( + final String host, + final int port, + final String username, + final String password, + final int thriftMaxFrameSize) { + super( + new SubscriptionSessionWrapper( + new SubscriptionTreeSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .thriftMaxFrameSize(thriftMaxFrameSize))); + } + + /////////////////////////////// open & close /////////////////////////////// + + @Override + public void open() throws IoTDBConnectionException { + super.open(); + } + + @Override + public void close() throws IoTDBConnectionException { + super.close(); + } + + /////////////////////////////// topic /////////////////////////////// + + @Override + public void createTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopic(topicName); + } + + @Override + public void createTopicIfNotExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopicIfNotExists(topicName); + } + + @Override + public void createTopic(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopic(topicName, properties); + } + + @Override + public void createTopicIfNotExists(final String topicName, final Properties properties) + throws IoTDBConnectionException, StatementExecutionException { + super.createTopicIfNotExists(topicName, properties); + } + + @Override + public void dropTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.dropTopic(topicName); + } + + @Override + public void dropTopicIfExists(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + super.dropTopicIfExists(topicName); + } + + @Override + public Set getTopics() throws IoTDBConnectionException, StatementExecutionException { + return super.getTopics(); + } + + @Override + public Optional getTopic(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + return super.getTopic(topicName); + } + + /////////////////////////////// subscription /////////////////////////////// + + @Override + public Set getSubscriptions() + throws IoTDBConnectionException, StatementExecutionException { + return super.getSubscriptions(); + } + + @Override + public Set getSubscriptions(final String topicName) + throws IoTDBConnectionException, StatementExecutionException { + return super.getSubscriptions(topicName); + } + + @Override + public void dropSubscription(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException { + super.dropSubscription(subscriptionId); + } + + @Override + public void dropSubscriptionIfExists(final String subscriptionId) + throws IoTDBConnectionException, StatementExecutionException { + super.dropSubscriptionIfExists(subscriptionId); + } + + /////////////////////////////// builder /////////////////////////////// + + @Deprecated // keep for forward compatibility + public static class Builder extends AbstractSessionBuilder { + + public Builder() { + // use tree model + super.sqlDialect = "tree"; + // disable auto fetch + super.enableAutoFetch = false; + // disable redirection + super.enableRedirection = false; + } + + public Builder host(final String host) { + super.host = host; + return this; + } + + public Builder port(final int port) { + super.rpcPort = port; + return this; + } + + public Builder username(final String username) { + super.username = username; + return this; + } + + public Builder password(final String password) { + super.pw = password; + return this; + } + + public Builder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + public ISubscriptionTreeSession build() { + return new SubscriptionTreeSession(this); + } + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTreeSessionBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTreeSessionBuilder.java new file mode 100644 index 0000000000000..dc21732ba971f --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionTreeSessionBuilder.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription; + +import org.apache.iotdb.session.AbstractSessionBuilder; + +public class SubscriptionTreeSessionBuilder extends AbstractSessionBuilder { + + public SubscriptionTreeSessionBuilder() { + // use tree model + super.sqlDialect = "tree"; + // disable auto fetch + super.enableAutoFetch = false; + // disable redirection + super.enableRedirection = false; + } + + public SubscriptionTreeSessionBuilder host(final String host) { + super.host = host; + return this; + } + + public SubscriptionTreeSessionBuilder port(final int port) { + super.rpcPort = port; + return this; + } + + public SubscriptionTreeSessionBuilder username(final String username) { + super.username = username; + return this; + } + + public SubscriptionTreeSessionBuilder password(final String password) { + super.pw = password; + return this; + } + + public SubscriptionTreeSessionBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + public ISubscriptionTreeSession build() { + return new SubscriptionTreeSession(this); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java new file mode 100644 index 0000000000000..0168a1ba3846d --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer; + +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +/** + * A subscription-based pull consumer interface for receiving messages from specified topics of + * table model. + */ +public interface ISubscriptionTablePullConsumer extends AutoCloseable { + + /** + * Opens this consumer to begin receiving messages. + * + * @throws SubscriptionException if the consumer fails to open + */ + void open() throws SubscriptionException; + + /** + * Subscribes to the specified topic. + * + * @param topicName the name of the topic to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String topicName) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames one or more topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String... topicNames) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames a set of topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final Set topicNames) throws SubscriptionException; + + /** + * Unsubscribes from the specified topic. + * + * @param topicName the name of the topic to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String topicName) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames one or more topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String... topicNames) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames a set of topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final Set topicNames) throws SubscriptionException; + + /** + * Retrieves messages from subscribed topics, waiting up to the specified timeout. + * + * @param timeout the maximum duration to wait for messages + * @return a list of received messages, which can be empty if no messages are available + * @throws SubscriptionException if polling fails + */ + List poll(final Duration timeout) throws SubscriptionException; + + /** + * Retrieves messages from subscribed topics, waiting up to the specified timeout in milliseconds. + * + * @param timeoutMs the maximum time in milliseconds to wait for messages + * @return a list of received messages, which can be empty if no messages are available + * @throws SubscriptionException if polling fails + */ + List poll(final long timeoutMs) throws SubscriptionException; + + /** + * Retrieves messages from the given set of topics, waiting up to the specified timeout. + * + * @param topicNames the set of topics to poll + * @param timeout the maximum duration to wait for messages + * @return a list of received messages, which can be empty if no messages are available + * @throws SubscriptionException if polling fails + */ + List poll(final Set topicNames, final Duration timeout) + throws SubscriptionException; + + /** + * Retrieves messages from the given set of topics, waiting up to the specified timeout in + * milliseconds. + * + * @param topicNames the set of topics to poll + * @param timeoutMs the maximum time in milliseconds to wait for messages + * @return a list of received messages, which can be empty if no messages are available + */ + List poll(final Set topicNames, final long timeoutMs); + + /** + * Commits a single message synchronously, indicating that it has been successfully processed. + * + * @param message the message to commit + * @throws SubscriptionException if the commit fails + */ + void commitSync(final SubscriptionMessage message) throws SubscriptionException; + + /** + * Commits multiple messages synchronously, indicating that they have been successfully processed. + * + * @param messages an iterable collection of messages to commit + * @throws SubscriptionException if the commit fails + */ + void commitSync(final Iterable messages) throws SubscriptionException; + + /** + * Commits a single message asynchronously, indicating that it has been successfully processed. + * + * @param message the message to commit + * @return a CompletableFuture that completes when the commit operation finishes + */ + CompletableFuture commitAsync(final SubscriptionMessage message); + + /** + * Commits multiple messages asynchronously, indicating that they have been successfully + * processed. + * + * @param messages an iterable collection of messages to commit + * @return a CompletableFuture that completes when the commit operation finishes + */ + CompletableFuture commitAsync(final Iterable messages); + + /** + * Commits a single message asynchronously, indicating that it has been successfully processed, + * and notifies the provided callback upon completion. + * + * @param message the message to commit + * @param callback a callback to receive the completion notification + */ + void commitAsync(final SubscriptionMessage message, final AsyncCommitCallback callback); + + /** + * Commits multiple messages asynchronously, indicating that they have been successfully + * processed, and notifies the provided callback upon completion. + * + * @param messages an iterable collection of messages to commit + * @param callback a callback to receive the completion notification + */ + void commitAsync( + final Iterable messages, final AsyncCommitCallback callback); + + /** + * Retrieves the unique identifier of this consumer. If no consumer ID was provided at the time of + * consumer construction, a random globally unique ID is automatically assigned after the consumer + * is opened. + * + * @return the unique consumer identifier + */ + String getConsumerId(); + + /** + * Retrieves the identifier of the consumer group to which this consumer belongs. If no consumer + * group ID was specified at the time of consumer construction, a random globally unique ID is + * automatically assigned after the consumer is opened. + * + * @return the consumer group's identifier + */ + String getConsumerGroupId(); + + /** + * Checks whether all topic messages have been consumed. + * + *

This method is used by the pull consumer in a loop that retrieves messages to determine if + * all messages for the subscription have been processed. It ensures that the consumer can + * correctly detect the termination signal for the subscription once all messages have been + * consumed. + * + * @return true if all topic messages have been consumed, false otherwise. + */ + boolean allTopicMessagesHaveBeenConsumed(); +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePushConsumer.java new file mode 100644 index 0000000000000..7b1dcb3836e69 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePushConsumer.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer; + +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import java.util.Set; + +/** + * A subscription-based push consumer interface for receiving messages from specified topics of + * table model. + */ +public interface ISubscriptionTablePushConsumer extends AutoCloseable { + + /** + * Opens this consumer to begin receiving messages. + * + * @throws SubscriptionException if the consumer fails to open + */ + void open() throws SubscriptionException; + + /** + * Subscribes to the specified topic. + * + * @param topicName the name of the topic to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String topicName) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames one or more topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String... topicNames) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames a set of topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final Set topicNames) throws SubscriptionException; + + /** + * Unsubscribes from the specified topic. + * + * @param topicName the name of the topic to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String topicName) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames one or more topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String... topicNames) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames a set of topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final Set topicNames) throws SubscriptionException; + + /** + * Retrieves the unique identifier of this consumer. If no consumer ID was provided at the time of + * consumer construction, a random globally unique ID is automatically assigned after the consumer + * is opened. + * + * @return the unique consumer identifier + */ + String getConsumerId(); + + /** + * Retrieves the identifier of the consumer group to which this consumer belongs. If no consumer + * group ID was specified at the time of consumer construction, a random globally unique ID is + * automatically assigned after the consumer is opened. + * + * @return the consumer group's identifier + */ + String getConsumerGroupId(); +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java new file mode 100644 index 0000000000000..803b7c51224a4 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer; + +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +/** + * A subscription-based pull consumer interface for receiving messages from specified topics of tree + * model. + */ +public interface ISubscriptionTreePullConsumer extends AutoCloseable { + + /** + * Opens this consumer to begin receiving messages. + * + * @throws SubscriptionException if the consumer fails to open + */ + void open() throws SubscriptionException; + + /** + * Subscribes to the specified topic. + * + * @param topicName the name of the topic to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String topicName) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames one or more topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String... topicNames) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames a set of topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final Set topicNames) throws SubscriptionException; + + /** + * Unsubscribes from the specified topic. + * + * @param topicName the name of the topic to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String topicName) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames one or more topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String... topicNames) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames a set of topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final Set topicNames) throws SubscriptionException; + + /** + * Retrieves messages from subscribed topics, waiting up to the specified timeout. + * + * @param timeout the maximum duration to wait for messages + * @return a list of received messages, which can be empty if no messages are available + * @throws SubscriptionException if polling fails + */ + List poll(final Duration timeout) throws SubscriptionException; + + /** + * Retrieves messages from subscribed topics, waiting up to the specified timeout in milliseconds. + * + * @param timeoutMs the maximum time in milliseconds to wait for messages + * @return a list of received messages, which can be empty if no messages are available + * @throws SubscriptionException if polling fails + */ + List poll(final long timeoutMs) throws SubscriptionException; + + /** + * Retrieves messages from the given set of topics, waiting up to the specified timeout. + * + * @param topicNames the set of topics to poll + * @param timeout the maximum duration to wait for messages + * @return a list of received messages, which can be empty if no messages are available + * @throws SubscriptionException if polling fails + */ + List poll(final Set topicNames, final Duration timeout) + throws SubscriptionException; + + /** + * Retrieves messages from the given set of topics, waiting up to the specified timeout in + * milliseconds. + * + * @param topicNames the set of topics to poll + * @param timeoutMs the maximum time in milliseconds to wait for messages + * @return a list of received messages, which can be empty if no messages are available + */ + List poll(final Set topicNames, final long timeoutMs); + + /** + * Commits a single message synchronously, indicating that it has been successfully processed. + * + * @param message the message to commit + * @throws SubscriptionException if the commit fails + */ + void commitSync(final SubscriptionMessage message) throws SubscriptionException; + + /** + * Commits multiple messages synchronously, indicating that they have been successfully processed. + * + * @param messages an iterable collection of messages to commit + * @throws SubscriptionException if the commit fails + */ + void commitSync(final Iterable messages) throws SubscriptionException; + + /** + * Commits a single message asynchronously, indicating that it has been successfully processed. + * + * @param message the message to commit + * @return a CompletableFuture that completes when the commit operation finishes + */ + CompletableFuture commitAsync(final SubscriptionMessage message); + + /** + * Commits multiple messages asynchronously, indicating that they have been successfully + * processed. + * + * @param messages an iterable collection of messages to commit + * @return a CompletableFuture that completes when the commit operation finishes + */ + CompletableFuture commitAsync(final Iterable messages); + + /** + * Commits a single message asynchronously, indicating that it has been successfully processed, + * and notifies the provided callback upon completion. + * + * @param message the message to commit + * @param callback a callback to receive the completion notification + */ + void commitAsync(final SubscriptionMessage message, final AsyncCommitCallback callback); + + /** + * Commits multiple messages asynchronously, indicating that they have been successfully + * processed, and notifies the provided callback upon completion. + * + * @param messages an iterable collection of messages to commit + * @param callback a callback to receive the completion notification + */ + void commitAsync( + final Iterable messages, final AsyncCommitCallback callback); + + /** + * Retrieves the unique identifier of this consumer. If no consumer ID was provided at the time of + * consumer construction, a random globally unique ID is automatically assigned after the consumer + * is opened. + * + * @return the unique consumer identifier + */ + String getConsumerId(); + + /** + * Retrieves the identifier of the consumer group to which this consumer belongs. If no consumer + * group ID was specified at the time of consumer construction, a random globally unique ID is + * automatically assigned after the consumer is opened. + * + * @return the consumer group's identifier + */ + String getConsumerGroupId(); + + /** + * Checks whether all topic messages have been consumed. + * + *

This method is used by the pull consumer in a loop that retrieves messages to determine if + * all messages for the subscription have been processed. It ensures that the consumer can + * correctly detect the termination signal for the subscription once all messages have been + * consumed. + * + * @return true if all topic messages have been consumed, false otherwise. + */ + boolean allTopicMessagesHaveBeenConsumed(); +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePushConsumer.java new file mode 100644 index 0000000000000..6eb371f23b2d7 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePushConsumer.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer; + +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import java.util.Set; + +/** + * A subscription-based push consumer interface for receiving messages from specified topics of tree + * model. + */ +public interface ISubscriptionTreePushConsumer extends AutoCloseable { + + /** + * Opens this consumer to begin receiving messages. + * + * @throws SubscriptionException if the consumer fails to open + */ + void open() throws SubscriptionException; + + /** + * Subscribes to the specified topic. + * + * @param topicName the name of the topic to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String topicName) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames one or more topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final String... topicNames) throws SubscriptionException; + + /** + * Subscribes to multiple topics. + * + * @param topicNames a set of topic names to subscribe to + * @throws SubscriptionException if the subscription cannot be established + */ + void subscribe(final Set topicNames) throws SubscriptionException; + + /** + * Unsubscribes from the specified topic. + * + * @param topicName the name of the topic to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String topicName) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames one or more topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final String... topicNames) throws SubscriptionException; + + /** + * Unsubscribes from multiple topics. + * + * @param topicNames a set of topic names to unsubscribe from + * @throws SubscriptionException if the unsubscription fails + */ + void unsubscribe(final Set topicNames) throws SubscriptionException; + + /** + * Retrieves the unique identifier of this consumer. If no consumer ID was provided at the time of + * consumer construction, a random globally unique ID is automatically assigned after the consumer + * is opened. + * + * @return the unique consumer identifier + */ + String getConsumerId(); + + /** + * Retrieves the identifier of the consumer group to which this consumer belongs. If no consumer + * group ID was specified at the time of consumer construction, a random globally unique ID is + * automatically assigned after the consumer is opened. + * + * @return the consumer group's identifier + */ + String getConsumerGroupId(); +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java deleted file mode 100644 index c1a89ded9837a..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ /dev/null @@ -1,1095 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.isession.SessionConfig; -import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; -import org.apache.iotdb.rpc.subscription.payload.poll.ErrorPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.FilePiecePayload; -import org.apache.iotdb.rpc.subscription.payload.poll.FileSealPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.PollFilePayload; -import org.apache.iotdb.rpc.subscription.payload.poll.PollPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequest; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequestType; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; -import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; -import org.apache.iotdb.session.subscription.util.IdentifierUtils; -import org.apache.iotdb.session.subscription.util.RandomStringGenerator; -import org.apache.iotdb.session.subscription.util.SubscriptionPollTimer; -import org.apache.iotdb.session.util.SessionUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.net.URLEncoder; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.LockSupport; -import java.util.stream.Collectors; - -import static org.apache.iotdb.rpc.subscription.config.TopicConstant.MODE_SNAPSHOT_VALUE; - -abstract class SubscriptionConsumer implements AutoCloseable { - - private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionConsumer.class); - - private static final long SLEEP_NS = 100_000_000L; // 100ms - - private final String username; - private final String password; - - protected String consumerId; - protected String consumerGroupId; - - private final long heartbeatIntervalMs; - private final long endpointsSyncIntervalMs; - - private final SubscriptionProviders providers; - - private final AtomicBoolean isClosed = new AtomicBoolean(true); - // This variable indicates whether the consumer has ever been closed. - private final AtomicBoolean isReleased = new AtomicBoolean(false); - - private final String fileSaveDir; - private final boolean fileSaveFsync; - - @SuppressWarnings("java:S3077") - protected volatile Map subscribedTopics = new HashMap<>(); - - public boolean allSnapshotTopicMessagesHaveBeenConsumed() { - return subscribedTopics.values().stream() - .noneMatch( - (config) -> config.getAttributesWithSourceMode().containsValue(MODE_SNAPSHOT_VALUE)); - } - - /////////////////////////////// getter /////////////////////////////// - - public String getConsumerId() { - return consumerId; - } - - public String getConsumerGroupId() { - return consumerGroupId; - } - - /////////////////////////////// ctor /////////////////////////////// - - protected SubscriptionConsumer(final Builder builder) { - final Set initialEndpoints = new HashSet<>(); - // From org.apache.iotdb.session.Session.getNodeUrls - // Priority is given to `host:port` over `nodeUrls`. - if (Objects.nonNull(builder.host) || Objects.nonNull(builder.port)) { - if (Objects.isNull(builder.host)) { - builder.host = SessionConfig.DEFAULT_HOST; - } - if (Objects.isNull(builder.port)) { - builder.port = SessionConfig.DEFAULT_PORT; - } - initialEndpoints.add(new TEndPoint(builder.host, builder.port)); - } else if (Objects.isNull(builder.nodeUrls)) { - builder.host = SessionConfig.DEFAULT_HOST; - builder.port = SessionConfig.DEFAULT_PORT; - initialEndpoints.add(new TEndPoint(builder.host, builder.port)); - } else { - initialEndpoints.addAll(SessionUtils.parseSeedNodeUrls(builder.nodeUrls)); - } - this.providers = new SubscriptionProviders(initialEndpoints); - - this.username = builder.username; - this.password = builder.password; - - this.consumerId = builder.consumerId; - this.consumerGroupId = builder.consumerGroupId; - - this.heartbeatIntervalMs = builder.heartbeatIntervalMs; - this.endpointsSyncIntervalMs = builder.endpointsSyncIntervalMs; - - this.fileSaveDir = builder.fileSaveDir; - this.fileSaveFsync = builder.fileSaveFsync; - } - - protected SubscriptionConsumer(final Builder builder, final Properties properties) { - this( - builder - .host( - (String) - properties.getOrDefault(ConsumerConstant.HOST_KEY, SessionConfig.DEFAULT_HOST)) - .port( - (Integer) - properties.getOrDefault(ConsumerConstant.PORT_KEY, SessionConfig.DEFAULT_PORT)) - .nodeUrls((List) properties.get(ConsumerConstant.NODE_URLS_KEY)) - .username( - (String) - properties.getOrDefault( - ConsumerConstant.USERNAME_KEY, SessionConfig.DEFAULT_USER)) - .password( - (String) - properties.getOrDefault( - ConsumerConstant.PASSWORD_KEY, SessionConfig.DEFAULT_PASSWORD)) - .consumerId((String) properties.get(ConsumerConstant.CONSUMER_ID_KEY)) - .consumerGroupId((String) properties.get(ConsumerConstant.CONSUMER_GROUP_ID_KEY)) - .heartbeatIntervalMs( - (Long) - properties.getOrDefault( - ConsumerConstant.HEARTBEAT_INTERVAL_MS_KEY, - ConsumerConstant.HEARTBEAT_INTERVAL_MS_DEFAULT_VALUE)) - .endpointsSyncIntervalMs( - (Long) - properties.getOrDefault( - ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_KEY, - ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_DEFAULT_VALUE)) - .fileSaveDir( - (String) - properties.getOrDefault( - ConsumerConstant.FILE_SAVE_DIR_KEY, - ConsumerConstant.FILE_SAVE_DIR_DEFAULT_VALUE)) - .fileSaveFsync( - (Boolean) - properties.getOrDefault( - ConsumerConstant.FILE_SAVE_FSYNC_KEY, - ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE))); - } - - /////////////////////////////// open & close /////////////////////////////// - - private void checkIfHasBeenClosed() throws SubscriptionException { - if (isReleased.get()) { - final String errorMessage = - String.format("%s has ever been closed, unsupported operation after closing.", this); - LOGGER.error(errorMessage); - throw new SubscriptionException(errorMessage); - } - } - - private void checkIfOpened() throws SubscriptionException { - if (isClosed.get()) { - final String errorMessage = - String.format("%s is not yet open, please open the subscription consumer first.", this); - LOGGER.error(errorMessage); - throw new SubscriptionException(errorMessage); - } - } - - public synchronized void open() throws SubscriptionException { - checkIfHasBeenClosed(); - - if (!isClosed.get()) { - return; - } - - // open subscription providers - providers.acquireWriteLock(); - try { - providers.openProviders(this); // throw SubscriptionException - } finally { - providers.releaseWriteLock(); - } - - // set isClosed to false before submitting workers - isClosed.set(false); - - // submit heartbeat worker - submitHeartbeatWorker(); - - // submit endpoints syncer - submitEndpointsSyncer(); - } - - @Override - public synchronized void close() { - if (isClosed.get()) { - return; - } - - // close subscription providers - providers.acquireWriteLock(); - providers.closeProviders(); - providers.releaseWriteLock(); - - isClosed.set(true); - - // mark is released to avoid reopening after closing - isReleased.set(true); - } - - boolean isClosed() { - return isClosed.get(); - } - - /////////////////////////////// subscribe & unsubscribe /////////////////////////////// - - public void subscribe(final String topicName) throws SubscriptionException { - subscribe(Collections.singleton(topicName)); - } - - public void subscribe(final String... topicNames) throws SubscriptionException { - subscribe(new HashSet<>(Arrays.asList(topicNames))); - } - - public void subscribe(final Set topicNames) throws SubscriptionException { - // parse topic names from external source - subscribe(topicNames, true); - } - - private void subscribe(Set topicNames, final boolean needParse) - throws SubscriptionException { - checkIfOpened(); - - if (needParse) { - topicNames = - topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); - } - - providers.acquireReadLock(); - try { - subscribeWithRedirection(topicNames); - } finally { - providers.releaseReadLock(); - } - } - - public void unsubscribe(final String topicName) throws SubscriptionException { - unsubscribe(Collections.singleton(topicName)); - } - - public void unsubscribe(final String... topicNames) throws SubscriptionException { - unsubscribe(new HashSet<>(Arrays.asList(topicNames))); - } - - public void unsubscribe(final Set topicNames) throws SubscriptionException { - // parse topic names from external source - unsubscribe(topicNames, true); - } - - private void unsubscribe(Set topicNames, final boolean needParse) - throws SubscriptionException { - checkIfOpened(); - - if (needParse) { - topicNames = - topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); - } - - providers.acquireReadLock(); - try { - unsubscribeWithRedirection(topicNames); - } finally { - providers.releaseReadLock(); - } - } - - /////////////////////////////// subscription provider /////////////////////////////// - - SubscriptionProvider constructProviderAndHandshake(final TEndPoint endPoint) - throws SubscriptionException { - final SubscriptionProvider provider = - new SubscriptionProvider( - endPoint, this.username, this.password, this.consumerId, this.consumerGroupId); - try { - provider.handshake(); - } catch (final Exception e) { - try { - provider.close(); - } catch (final Exception ignored) { - } - throw new SubscriptionConnectionException( - String.format("Failed to handshake with subscription provider %s", provider)); - } - - // update consumer id and consumer group id if not exist - if (Objects.isNull(this.consumerId)) { - this.consumerId = provider.getConsumerId(); - } - if (Objects.isNull(this.consumerGroupId)) { - this.consumerGroupId = provider.getConsumerGroupId(); - } - - return provider; - } - - /////////////////////////////// file ops /////////////////////////////// - - private Path getFileDir(final String topicName) throws IOException { - final Path dirPath = - Paths.get(fileSaveDir).resolve(consumerGroupId).resolve(consumerId).resolve(topicName); - Files.createDirectories(dirPath); - return dirPath; - } - - private Path getFilePath( - final String topicName, - final String fileName, - final boolean allowFileAlreadyExistsException, - final boolean allowInvalidPathException) - throws SubscriptionException { - try { - final Path filePath = getFileDir(topicName).resolve(fileName); - Files.createFile(filePath); - return filePath; - } catch (final FileAlreadyExistsException fileAlreadyExistsException) { - if (allowFileAlreadyExistsException) { - final String suffix = RandomStringGenerator.generate(16); - LOGGER.warn( - "Detect already existed file {} when polling topic {}, add random suffix {} to filename", - fileName, - topicName, - suffix); - return getFilePath(topicName, fileName + "." + suffix, false, true); - } - throw new SubscriptionRuntimeNonCriticalException( - fileAlreadyExistsException.getMessage(), fileAlreadyExistsException); - } catch (final InvalidPathException invalidPathException) { - if (allowInvalidPathException) { - return getFilePath(URLEncoder.encode(topicName), fileName, true, false); - } - throw new SubscriptionRuntimeNonCriticalException( - invalidPathException.getMessage(), invalidPathException); - } catch (final IOException e) { - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - } - - /////////////////////////////// poll /////////////////////////////// - - protected List poll( - /* @NotNull */ final Set topicNames, final long timeoutMs) - throws SubscriptionException { - // check topic names - if (subscribedTopics.isEmpty()) { - LOGGER.info("SubscriptionConsumer {} has not subscribed to any topics yet", this); - return Collections.emptyList(); - } - - topicNames.stream() - .filter(topicName -> !subscribedTopics.containsKey(topicName)) - .forEach( - topicName -> - LOGGER.warn( - "SubscriptionConsumer {} does not subscribe to topic {}", this, topicName)); - - final List messages = new ArrayList<>(); - final SubscriptionPollTimer timer = - new SubscriptionPollTimer(System.currentTimeMillis(), timeoutMs); - - do { - try { - // poll tablets or file - for (final SubscriptionPollResponse pollResponse : pollInternal(topicNames)) { - final short responseType = pollResponse.getResponseType(); - if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) { - LOGGER.warn("unexpected response type: {}", responseType); - continue; - } - switch (SubscriptionPollResponseType.valueOf(responseType)) { - case TABLETS: - messages.add( - new SubscriptionMessage( - pollResponse.getCommitContext(), - ((TabletsPayload) pollResponse.getPayload()).getTablets())); - break; - case FILE_INIT: - pollFile( - pollResponse.getCommitContext(), - ((FileInitPayload) pollResponse.getPayload()).getFileName()) - .ifPresent(messages::add); - break; - case ERROR: - final ErrorPayload payload = (ErrorPayload) pollResponse.getPayload(); - final String errorMessage = payload.getErrorMessage(); - if (payload.isCritical()) { - throw new SubscriptionRuntimeCriticalException(errorMessage); - } else { - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - case TERMINATION: - final SubscriptionCommitContext commitContext = pollResponse.getCommitContext(); - final String topicNameToUnsubscribe = commitContext.getTopicName(); - LOGGER.info( - "Termination occurred when SubscriptionConsumer {} polling topics {}, unsubscribe topic {} automatically", - this, - topicNames, - topicNameToUnsubscribe); - unsubscribe(Collections.singleton(topicNameToUnsubscribe), false); - break; - default: - LOGGER.warn("unexpected response type: {}", responseType); - break; - } - } - } catch (final SubscriptionRuntimeNonCriticalException e) { - LOGGER.warn( - "SubscriptionRuntimeNonCriticalException occurred when SubscriptionConsumer {} polling topics {}", - this, - topicNames, - e); - // nack and clear messages - try { - nack(messages); - messages.clear(); - } catch (final Exception ignored) { - } - } catch (final SubscriptionRuntimeCriticalException e) { - LOGGER.warn( - "SubscriptionRuntimeCriticalException occurred when SubscriptionConsumer {} polling topics {}", - this, - topicNames, - e); - // nack and clear messages - try { - nack(messages); - messages.clear(); - } catch (final Exception ignored) { - } - // rethrow - throw e; - } - if (!messages.isEmpty()) { - return messages; - } - // update timer - timer.update(); - // TODO: associated with timeoutMs instead of hardcoding - LockSupport.parkNanos(SLEEP_NS); // wait some time - } while (timer.notExpired()); - - LOGGER.info( - "SubscriptionConsumer {} poll empty message after {} millisecond(s)", this, timeoutMs); - return messages; - } - - private Optional pollFile( - final SubscriptionCommitContext commitContext, final String fileName) - throws SubscriptionException { - final String topicName = commitContext.getTopicName(); - final Path filePath = getFilePath(topicName, fileName, true, true); - final File file = filePath.toFile(); - try (final RandomAccessFile fileWriter = new RandomAccessFile(file, "rw")) { - return Optional.of(pollFileInternal(commitContext, file, fileWriter)); - } catch (final Exception e) { - // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(commitContext, file.getAbsolutePath()))); - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - } - - private SubscriptionMessage pollFileInternal( - final SubscriptionCommitContext commitContext, - final File file, - final RandomAccessFile fileWriter) - throws IOException, SubscriptionException { - final int dataNodeId = commitContext.getDataNodeId(); - final String topicName = commitContext.getTopicName(); - final String fileName = file.getName(); - - LOGGER.info( - "{} start to poll file {} with commit context {}", - this, - file.getAbsolutePath(), - commitContext); - - long writingOffset = fileWriter.length(); - while (true) { - final List responses = - pollFileInternal(dataNodeId, topicName, fileName, writingOffset); - - // It's agreed that the server will always return at least one response, even in case of - // failure. - if (responses.isEmpty()) { - final String errorMessage = - String.format("SubscriptionConsumer %s poll empty response", this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // Only one SubscriptionEvent polled currently... - final SubscriptionPollResponse response = responses.get(0); - final SubscriptionPollPayload payload = response.getPayload(); - final short responseType = response.getResponseType(); - if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) { - final String errorMessage = String.format("unexpected response type: %s", responseType); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - switch (SubscriptionPollResponseType.valueOf(responseType)) { - case FILE_PIECE: - { - // check commit context - final SubscriptionCommitContext incomingCommitContext = response.getCommitContext(); - if (Objects.isNull(incomingCommitContext) - || !Objects.equals(commitContext, incomingCommitContext)) { - final String errorMessage = - String.format( - "inconsistent commit context, current is %s, incoming is %s, consumer: %s", - commitContext, incomingCommitContext, this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // check file name - if (!fileName.startsWith(((FilePiecePayload) payload).getFileName())) { - final String errorMessage = - String.format( - "inconsistent file name, current is %s, incoming is %s, consumer: %s", - fileName, ((FilePiecePayload) payload).getFileName(), this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // write file piece - fileWriter.write(((FilePiecePayload) payload).getFilePiece()); - if (fileSaveFsync) { - fileWriter.getFD().sync(); - } - - // check offset - if (!Objects.equals( - fileWriter.length(), ((FilePiecePayload) payload).getNextWritingOffset())) { - final String errorMessage = - String.format( - "inconsistent file offset, current is %s, incoming is %s, consumer: %s", - fileWriter.length(), - ((FilePiecePayload) payload).getNextWritingOffset(), - this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // update offset - writingOffset = ((FilePiecePayload) payload).getNextWritingOffset(); - break; - } - case FILE_SEAL: - { - // check commit context - final SubscriptionCommitContext incomingCommitContext = response.getCommitContext(); - if (Objects.isNull(incomingCommitContext) - || !Objects.equals(commitContext, incomingCommitContext)) { - final String errorMessage = - String.format( - "inconsistent commit context, current is %s, incoming is %s, consumer: %s", - commitContext, incomingCommitContext, this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // check file name - if (!fileName.startsWith(((FileSealPayload) payload).getFileName())) { - final String errorMessage = - String.format( - "inconsistent file name, current is %s, incoming is %s, consumer: %s", - fileName, ((FileSealPayload) payload).getFileName(), this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // check file length - if (fileWriter.length() != ((FileSealPayload) payload).getFileLength()) { - final String errorMessage = - String.format( - "inconsistent file length, current is %s, incoming is %s, consumer: %s", - fileWriter.length(), ((FileSealPayload) payload).getFileLength(), this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - - // optional sync and close - if (fileSaveFsync) { - fileWriter.getFD().sync(); - } - fileWriter.close(); - - LOGGER.info( - "SubscriptionConsumer {} successfully poll file {} with commit context {}", - this, - file.getAbsolutePath(), - commitContext); - - // generate subscription message - return new SubscriptionMessage(commitContext, file.getAbsolutePath()); - } - case ERROR: - { - // no need to check commit context - - final String errorMessage = ((ErrorPayload) payload).getErrorMessage(); - final boolean critical = ((ErrorPayload) payload).isCritical(); - LOGGER.warn( - "Error occurred when SubscriptionConsumer {} polling file {} with commit context {}: {}, critical: {}", - this, - file.getAbsolutePath(), - commitContext, - errorMessage, - critical); - if (critical) { - throw new SubscriptionRuntimeCriticalException(errorMessage); - } else { - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - } - default: - final String errorMessage = String.format("unexpected response type: %s", responseType); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - } - } - - private List pollInternal(final Set topicNames) - throws SubscriptionException { - providers.acquireReadLock(); - try { - final SubscriptionProvider provider = providers.getNextAvailableProvider(); - if (Objects.isNull(provider) || !provider.isAvailable()) { - if (isClosed()) { - return Collections.emptyList(); - } - throw new SubscriptionConnectionException( - String.format( - "Cluster has no available subscription providers when %s poll topic %s", - this, topicNames)); - } - // ignore SubscriptionConnectionException to improve poll auto retry - try { - return provider.poll( - new SubscriptionPollRequest( - SubscriptionPollRequestType.POLL.getType(), new PollPayload(topicNames), 0L)); - } catch (final SubscriptionConnectionException ignored) { - return Collections.emptyList(); - } - } finally { - providers.releaseReadLock(); - } - } - - private List pollFileInternal( - final int dataNodeId, final String topicName, final String fileName, final long writingOffset) - throws SubscriptionException { - providers.acquireReadLock(); - try { - final SubscriptionProvider provider = providers.getProvider(dataNodeId); - if (Objects.isNull(provider) || !provider.isAvailable()) { - if (isClosed()) { - return Collections.emptyList(); - } - throw new SubscriptionConnectionException( - String.format( - "something unexpected happened when %s poll file from subscription provider with data node id %s, the subscription provider may be unavailable or not existed", - this, dataNodeId)); - } - // ignore SubscriptionConnectionException to improve poll auto retry - try { - return provider.poll( - new SubscriptionPollRequest( - SubscriptionPollRequestType.POLL_FILE.getType(), - new PollFilePayload(topicName, fileName, writingOffset), - 0L)); - } catch (final SubscriptionConnectionException ignored) { - return Collections.emptyList(); - } - } finally { - providers.releaseReadLock(); - } - } - - /////////////////////////////// commit sync (ack & nack) /////////////////////////////// - - protected void ack(final Iterable messages) throws SubscriptionException { - final Map> dataNodeIdToSubscriptionCommitContexts = - new HashMap<>(); - for (final SubscriptionMessage message : messages) { - dataNodeIdToSubscriptionCommitContexts - .computeIfAbsent(message.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) - .add(message.getCommitContext()); - } - for (final Map.Entry> entry : - dataNodeIdToSubscriptionCommitContexts.entrySet()) { - commitInternal(entry.getKey(), entry.getValue(), false); - } - } - - protected void nack(final Iterable messages) throws SubscriptionException { - final Map> dataNodeIdToSubscriptionCommitContexts = - new HashMap<>(); - for (final SubscriptionMessage message : messages) { - // make every effort to delete stale intermediate file - if (Objects.equals( - SubscriptionMessageType.TS_FILE_HANDLER.getType(), message.getMessageType())) { - try { - message.getTsFileHandler().deleteFile(); - } catch (final Exception ignored) { - } - } - dataNodeIdToSubscriptionCommitContexts - .computeIfAbsent(message.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) - .add(message.getCommitContext()); - } - for (final Map.Entry> entry : - dataNodeIdToSubscriptionCommitContexts.entrySet()) { - commitInternal(entry.getKey(), entry.getValue(), true); - } - } - - private void commitInternal( - final int dataNodeId, - final List subscriptionCommitContexts, - final boolean nack) - throws SubscriptionException { - providers.acquireReadLock(); - try { - final SubscriptionProvider provider = providers.getProvider(dataNodeId); - if (Objects.isNull(provider) || !provider.isAvailable()) { - if (isClosed()) { - return; - } - throw new SubscriptionConnectionException( - String.format( - "something unexpected happened when %s commit (nack: %s) messages to subscription provider with data node id %s, the subscription provider may be unavailable or not existed", - this, nack, dataNodeId)); - } - provider.commit(subscriptionCommitContexts, nack); - } finally { - providers.releaseReadLock(); - } - } - - /////////////////////////////// heartbeat /////////////////////////////// - - private void submitHeartbeatWorker() { - final ScheduledFuture[] future = new ScheduledFuture[1]; - future[0] = - SubscriptionExecutorServiceManager.submitHeartbeatWorker( - () -> { - if (isClosed()) { - if (Objects.nonNull(future[0])) { - future[0].cancel(false); - LOGGER.info("SubscriptionConsumer {} cancel heartbeat worker", this); - } - return; - } - providers.heartbeat(this); - }, - heartbeatIntervalMs); - LOGGER.info("SubscriptionConsumer {} submit heartbeat worker", this); - } - - /////////////////////////////// sync endpoints /////////////////////////////// - - private void submitEndpointsSyncer() { - final ScheduledFuture[] future = new ScheduledFuture[1]; - future[0] = - SubscriptionExecutorServiceManager.submitEndpointsSyncer( - () -> { - if (isClosed()) { - if (Objects.nonNull(future[0])) { - future[0].cancel(false); - LOGGER.info("SubscriptionConsumer {} cancel endpoints syncer", this); - } - return; - } - providers.sync(this); - }, - endpointsSyncIntervalMs); - LOGGER.info("SubscriptionConsumer {} submit endpoints syncer", this); - } - - /////////////////////////////// commit async /////////////////////////////// - - protected void commitAsync( - final Iterable messages, final AsyncCommitCallback callback) { - SubscriptionExecutorServiceManager.submitAsyncCommitWorker( - new AsyncCommitWorker(messages, callback)); - } - - private class AsyncCommitWorker implements Runnable { - - private final Iterable messages; - private final AsyncCommitCallback callback; - - public AsyncCommitWorker( - final Iterable messages, final AsyncCommitCallback callback) { - this.messages = messages; - this.callback = callback; - } - - @Override - public void run() { - if (isClosed()) { - return; - } - - try { - ack(messages); - callback.onComplete(); - } catch (final Exception e) { - callback.onFailure(e); - } - } - } - - protected CompletableFuture commitAsync(final Iterable messages) { - final CompletableFuture future = new CompletableFuture<>(); - SubscriptionExecutorServiceManager.submitAsyncCommitWorker( - () -> { - if (isClosed()) { - return; - } - - try { - ack(messages); - future.complete(null); - } catch (final Throwable e) { - future.completeExceptionally(e); - } - }); - return future; - } - - /////////////////////////////// redirection /////////////////////////////// - - private void subscribeWithRedirection(final Set topicNames) throws SubscriptionException { - final List providers = this.providers.getAllAvailableProviders(); - if (providers.isEmpty()) { - throw new SubscriptionConnectionException( - String.format( - "Cluster has no available subscription providers when %s subscribe topic %s", - this, topicNames)); - } - for (final SubscriptionProvider provider : providers) { - try { - subscribedTopics = provider.subscribe(topicNames); - return; - } catch (final Exception e) { - LOGGER.warn( - "{} failed to subscribe topics {} from subscription provider {}, try next subscription provider...", - this, - topicNames, - provider, - e); - } - } - final String errorMessage = - String.format( - "%s failed to subscribe topics %s from all available subscription providers %s", - this, topicNames, providers); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeCriticalException(errorMessage); - } - - private void unsubscribeWithRedirection(final Set topicNames) - throws SubscriptionException { - final List providers = this.providers.getAllAvailableProviders(); - if (providers.isEmpty()) { - throw new SubscriptionConnectionException( - String.format( - "Cluster has no available subscription providers when %s unsubscribe topic %s", - this, topicNames)); - } - for (final SubscriptionProvider provider : providers) { - try { - subscribedTopics = provider.unsubscribe(topicNames); - return; - } catch (final Exception e) { - LOGGER.warn( - "{} failed to unsubscribe topics {} from subscription provider {}, try next subscription provider...", - this, - topicNames, - provider, - e); - } - } - final String errorMessage = - String.format( - "%s failed to unsubscribe topics %s from all available subscription providers %s", - this, topicNames, providers); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeCriticalException(errorMessage); - } - - Map fetchAllEndPointsWithRedirection() throws SubscriptionException { - final List providers = this.providers.getAllAvailableProviders(); - if (providers.isEmpty()) { - throw new SubscriptionConnectionException( - String.format( - "Cluster has no available subscription providers when %s fetch all endpoints", this)); - } - for (final SubscriptionProvider provider : providers) { - try { - return provider.getSessionConnection().fetchAllEndPoints(); - } catch (final Exception e) { - LOGGER.warn( - "{} failed to fetch all endpoints from subscription provider {}, try next subscription provider...", - this, - provider, - e); - } - } - final String errorMessage = - String.format( - "%s failed to fetch all endpoints from all available subscription providers %s", - this, providers); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeCriticalException(errorMessage); - } - - /////////////////////////////// builder /////////////////////////////// - - public abstract static class Builder { - - protected String host; - protected Integer port; - protected List nodeUrls; - - protected String username = SessionConfig.DEFAULT_USER; - protected String password = SessionConfig.DEFAULT_PASSWORD; - - protected String consumerId; - protected String consumerGroupId; - - protected long heartbeatIntervalMs = ConsumerConstant.HEARTBEAT_INTERVAL_MS_DEFAULT_VALUE; - protected long endpointsSyncIntervalMs = - ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_DEFAULT_VALUE; - - protected String fileSaveDir = ConsumerConstant.FILE_SAVE_DIR_DEFAULT_VALUE; - protected boolean fileSaveFsync = ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE; - - public Builder host(final String host) { - this.host = host; - return this; - } - - public Builder port(final int port) { - this.port = port; - return this; - } - - public Builder nodeUrls(final List nodeUrls) { - this.nodeUrls = nodeUrls; - return this; - } - - public Builder username(final String username) { - this.username = username; - return this; - } - - public Builder password(final String password) { - this.password = password; - return this; - } - - public Builder consumerId(final String consumerId) { - this.consumerId = IdentifierUtils.parseIdentifier(consumerId); - return this; - } - - public Builder consumerGroupId(final String consumerGroupId) { - this.consumerGroupId = IdentifierUtils.parseIdentifier(consumerGroupId); - return this; - } - - public Builder heartbeatIntervalMs(final long heartbeatIntervalMs) { - this.heartbeatIntervalMs = - Math.max(heartbeatIntervalMs, ConsumerConstant.HEARTBEAT_INTERVAL_MS_MIN_VALUE); - return this; - } - - public Builder endpointsSyncIntervalMs(final long endpointsSyncIntervalMs) { - this.endpointsSyncIntervalMs = - Math.max(endpointsSyncIntervalMs, ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE); - return this; - } - - public Builder fileSaveDir(final String fileSaveDir) { - this.fileSaveDir = fileSaveDir; - return this; - } - - public Builder fileSaveFsync(final boolean fileSaveFsync) { - this.fileSaveFsync = fileSaveFsync; - return this; - } - - public abstract SubscriptionPullConsumer buildPullConsumer(); - - public abstract SubscriptionPushConsumer buildPushConsumer(); - } - - /////////////////////////////// stringify /////////////////////////////// - - protected Map coreReportMessage() { - final Map result = new HashMap<>(5); - result.put("consumerId", consumerId); - result.put("consumerGroupId", consumerGroupId); - result.put("isClosed", isClosed.toString()); - result.put("fileSaveDir", fileSaveDir); - result.put("subscribedTopicNames", subscribedTopics.keySet().toString()); - return result; - } - - protected Map allReportMessage() { - final Map result = new HashMap<>(10); - result.put("consumerId", consumerId); - result.put("consumerGroupId", consumerGroupId); - result.put("heartbeatIntervalMs", String.valueOf(heartbeatIntervalMs)); - result.put("endpointsSyncIntervalMs", String.valueOf(endpointsSyncIntervalMs)); - result.put("providers", providers.toString()); - result.put("isClosed", isClosed.toString()); - result.put("isReleased", isReleased.toString()); - result.put("fileSaveDir", fileSaveDir); - result.put("fileSaveFsync", String.valueOf(fileSaveFsync)); - result.put("subscribedTopics", subscribedTopics.toString()); - return result; - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java deleted file mode 100644 index 6ce5946d4fcc6..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public final class SubscriptionExecutorServiceManager { - - private static final Logger LOGGER = - LoggerFactory.getLogger(SubscriptionExecutorServiceManager.class); - - private static final long AWAIT_TERMINATION_TIMEOUT_MS = 15_000L; - - private static final String CONTROL_FLOW_EXECUTOR_NAME = "SubscriptionControlFlowExecutor"; - private static final String UPSTREAM_DATA_FLOW_EXECUTOR_NAME = - "SubscriptionUpstreamDataFlowExecutor"; - private static final String DOWNSTREAM_DATA_FLOW_EXECUTOR_NAME = - "SubscriptionDownstreamDataFlowExecutor"; - - /** - * Control Flow Executor: execute heartbeat worker and endpoints syncer for {@link - * SubscriptionConsumer} - */ - private static final SubscriptionExecutorService CONTROL_FLOW_EXECUTOR = - new SubscriptionExecutorService( - CONTROL_FLOW_EXECUTOR_NAME, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)); - - /** - * Upstream Data Flow Executor: execute auto commit worker and async commit worker for {@link - * SubscriptionPullConsumer} - */ - private static final SubscriptionExecutorService UPSTREAM_DATA_FLOW_EXECUTOR = - new SubscriptionExecutorService( - UPSTREAM_DATA_FLOW_EXECUTOR_NAME, - Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)); - - /** - * Downstream Data Flow Executor: execute auto poll worker for {@link SubscriptionPushConsumer} - */ - private static final SubscriptionExecutorService DOWNSTREAM_DATA_FLOW_EXECUTOR = - new SubscriptionExecutorService( - DOWNSTREAM_DATA_FLOW_EXECUTOR_NAME, - Math.max(Runtime.getRuntime().availableProcessors(), 1)); - - /////////////////////////////// set core pool size /////////////////////////////// - - public static void setControlFlowExecutorCorePoolSize(final int corePoolSize) { - CONTROL_FLOW_EXECUTOR.setCorePoolSize(corePoolSize); - } - - public static void setUpstreamDataFlowExecutorCorePoolSize(final int corePoolSize) { - UPSTREAM_DATA_FLOW_EXECUTOR.setCorePoolSize(corePoolSize); - } - - public static void setDownstreamDataFlowExecutorCorePoolSize(final int corePoolSize) { - DOWNSTREAM_DATA_FLOW_EXECUTOR.setCorePoolSize(corePoolSize); - } - - /////////////////////////////// shutdown hook /////////////////////////////// - - static { - // register shutdown hook - Runtime.getRuntime() - .addShutdownHook( - new Thread( - new SubscriptionExecutorServiceShutdownHook(), - "SubscriptionExecutorServiceShutdownHook")); - } - - private static class SubscriptionExecutorServiceShutdownHook implements Runnable { - - @Override - public void run() { - // shutdown executors - CONTROL_FLOW_EXECUTOR.shutdown(); - UPSTREAM_DATA_FLOW_EXECUTOR.shutdown(); - DOWNSTREAM_DATA_FLOW_EXECUTOR.shutdown(); - } - } - - /////////////////////////////// submitter /////////////////////////////// - - @SuppressWarnings("unsafeThreadSchedule") - public static ScheduledFuture submitHeartbeatWorker( - final Runnable task, final long heartbeatIntervalMs) { - CONTROL_FLOW_EXECUTOR.launchIfNeeded(); - return CONTROL_FLOW_EXECUTOR.scheduleWithFixedDelay( - task, - generateRandomInitialDelayMs(heartbeatIntervalMs), - heartbeatIntervalMs, - TimeUnit.MILLISECONDS); - } - - @SuppressWarnings("unsafeThreadSchedule") - public static ScheduledFuture submitEndpointsSyncer( - final Runnable task, final long endpointsSyncIntervalMs) { - CONTROL_FLOW_EXECUTOR.launchIfNeeded(); - return CONTROL_FLOW_EXECUTOR.scheduleWithFixedDelay( - task, - generateRandomInitialDelayMs(endpointsSyncIntervalMs), - endpointsSyncIntervalMs, - TimeUnit.MILLISECONDS); - } - - @SuppressWarnings("unsafeThreadSchedule") - public static ScheduledFuture submitAutoCommitWorker( - final Runnable task, final long autoCommitIntervalMs) { - UPSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); - return UPSTREAM_DATA_FLOW_EXECUTOR.scheduleWithFixedDelay( - task, - generateRandomInitialDelayMs(autoCommitIntervalMs), - autoCommitIntervalMs, - TimeUnit.MILLISECONDS); - } - - public static void submitAsyncCommitWorker(final Runnable task) { - UPSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); - UPSTREAM_DATA_FLOW_EXECUTOR.submit(task); - } - - @SuppressWarnings("unsafeThreadSchedule") - public static ScheduledFuture submitAutoPollWorker( - final Runnable task, final long autoPollIntervalMs) { - DOWNSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); - return DOWNSTREAM_DATA_FLOW_EXECUTOR.scheduleWithFixedDelay( - task, - generateRandomInitialDelayMs(autoPollIntervalMs), - autoPollIntervalMs, - TimeUnit.MILLISECONDS); - } - - /////////////////////////////// subscription executor service /////////////////////////////// - - private static class SubscriptionExecutorService { - - String name; - volatile int corePoolSize; - volatile ScheduledExecutorService executor; - - SubscriptionExecutorService(final String name, final int corePoolSize) { - this.name = name; - this.corePoolSize = corePoolSize; - } - - boolean isShutdown() { - return Objects.isNull(this.executor); - } - - void setCorePoolSize(final int corePoolSize) { - if (isShutdown()) { - synchronized (this) { - if (isShutdown()) { - this.corePoolSize = corePoolSize; - return; - } - } - } - - LOGGER.warn( - "{} has been launched, set core pool size to {} will be ignored", - this.name, - corePoolSize); - } - - void launchIfNeeded() { - if (isShutdown()) { - synchronized (this) { - if (isShutdown()) { - LOGGER.info("Launching {} with core pool size {}...", this.name, this.corePoolSize); - - this.executor = - Executors.newScheduledThreadPool( - this.corePoolSize, - r -> { - final Thread t = - new Thread(Thread.currentThread().getThreadGroup(), r, this.name, 0); - if (!t.isDaemon()) { - t.setDaemon(true); - } - if (t.getPriority() != Thread.NORM_PRIORITY) { - t.setPriority(Thread.NORM_PRIORITY); - } - return t; - }); - } - } - } - } - - void shutdown() { - if (!isShutdown()) { - synchronized (this) { - if (!isShutdown()) { - LOGGER.info("Shutting down {}...", this.name); - - this.executor.shutdown(); - try { - if (!this.executor.awaitTermination( - AWAIT_TERMINATION_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { - this.executor.shutdownNow(); - LOGGER.warn( - "Interrupt the worker, which may cause some task inconsistent. Please check the biz logs."); - if (!this.executor.awaitTermination( - AWAIT_TERMINATION_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { - LOGGER.error( - "Thread pool can't be shutdown even with interrupting worker threads, which may cause some task inconsistent. Please check the biz logs."); - } - } - } catch (final InterruptedException e) { - this.executor.shutdownNow(); - LOGGER.error( - "The current thread is interrupted when it is trying to stop the worker threads. This may leave an inconsistent state. Please check the biz logs."); - Thread.currentThread().interrupt(); - } - - this.executor = null; - } - } - } - } - - @SuppressWarnings("unsafeThreadSchedule") - ScheduledFuture scheduleWithFixedDelay( - final Runnable task, final long initialDelay, final long delay, final TimeUnit unit) { - if (!isShutdown()) { - synchronized (this) { - if (!isShutdown()) { - return this.executor.scheduleWithFixedDelay(task, initialDelay, delay, unit); - } - } - } - - LOGGER.warn("{} has not been launched, ignore scheduleWithFixedDelay for task", this.name); - return null; - } - - Future submit(final Runnable task) { - if (!isShutdown()) { - synchronized (this) { - if (!isShutdown()) { - return this.executor.submit(task); - } - } - } - - LOGGER.warn("{} has not been launched, ignore submit task", this.name); - return null; - } - } - - /////////////////////////////// utility /////////////////////////////// - - private static long generateRandomInitialDelayMs(final long maxMs) { - return (long) (Math.random() * maxMs); - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java deleted file mode 100644 index ba99774590b55..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; -import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequest; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeCloseReq; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeCommitReq; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHandshakeReq; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHeartbeatReq; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribePollReq; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeSubscribeReq; -import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeUnsubscribeReq; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHandshakeResp; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribePollResp; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeSubscribeResp; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeUnsubscribeResp; -import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; -import org.apache.iotdb.session.subscription.SubscriptionSession; -import org.apache.iotdb.session.subscription.SubscriptionSessionConnection; - -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - -final class SubscriptionProvider extends SubscriptionSession { - - private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionProvider.class); - - private String consumerId; - private String consumerGroupId; - - private final AtomicBoolean isClosed = new AtomicBoolean(true); - private final AtomicBoolean isAvailable = new AtomicBoolean(false); - - private final TEndPoint endPoint; - private int dataNodeId; - - SubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String consumerId, - final String consumerGroupId) { - super(endPoint.ip, endPoint.port, username, password); - - this.endPoint = endPoint; - this.consumerId = consumerId; - this.consumerGroupId = consumerGroupId; - } - - boolean isAvailable() { - return isAvailable.get(); - } - - void setAvailable() { - isAvailable.set(true); - } - - void setUnavailable() { - isAvailable.set(false); - } - - int getDataNodeId() { - return dataNodeId; - } - - String getConsumerId() { - return consumerId; - } - - String getConsumerGroupId() { - return consumerGroupId; - } - - TEndPoint getEndPoint() { - return endPoint; - } - - SubscriptionSessionConnection getSessionConnection() { - return (SubscriptionSessionConnection) defaultSessionConnection; - } - - /////////////////////////////// open & close /////////////////////////////// - - synchronized void handshake() throws SubscriptionException, IoTDBConnectionException { - if (!isClosed.get()) { - return; - } - - super.open(); // throw IoTDBConnectionException - - // TODO: pass the complete consumer parameter configuration to the server - final Map consumerAttributes = new HashMap<>(); - consumerAttributes.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, consumerGroupId); - consumerAttributes.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); - - final PipeSubscribeHandshakeResp resp = - handshake(new ConsumerConfig(consumerAttributes)); // throw SubscriptionException - dataNodeId = resp.getDataNodeId(); - consumerId = resp.getConsumerId(); - consumerGroupId = resp.getConsumerGroupId(); - - isClosed.set(false); - setAvailable(); - } - - PipeSubscribeHandshakeResp handshake(final ConsumerConfig consumerConfig) - throws SubscriptionException { - final PipeSubscribeHandshakeReq req; - try { - req = PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig); - } catch (final IOException e) { - LOGGER.warn( - "IOException occurred when SubscriptionProvider {} serialize handshake request {}", - this, - consumerConfig, - e); - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(req); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} handshake with request {}, set SubscriptionProvider unavailable", - this, - consumerConfig, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - return PipeSubscribeHandshakeResp.fromTPipeSubscribeResp(resp); - } - - @Override - public synchronized void close() throws SubscriptionException, IoTDBConnectionException { - if (isClosed.get()) { - return; - } - - try { - closeInternal(); // throw SubscriptionException - } finally { - super.close(); // throw IoTDBConnectionException - setUnavailable(); - isClosed.set(true); - } - } - - void closeInternal() throws SubscriptionException { - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(PipeSubscribeCloseReq.toTPipeSubscribeReq()); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} close, set SubscriptionProvider unavailable", - this, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - } - - /////////////////////////////// subscription APIs /////////////////////////////// - - void heartbeat() throws SubscriptionException { - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(PipeSubscribeHeartbeatReq.toTPipeSubscribeReq()); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} heartbeat, set SubscriptionProvider unavailable", - this, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - } - - Map subscribe(final Set topicNames) throws SubscriptionException { - final PipeSubscribeSubscribeReq req; - try { - req = PipeSubscribeSubscribeReq.toTPipeSubscribeReq(topicNames); - } catch (final IOException e) { - LOGGER.warn( - "IOException occurred when SubscriptionProvider {} serialize subscribe request {}", - this, - topicNames, - e); - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(req); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} subscribe with request {}, set SubscriptionProvider unavailable", - this, - topicNames, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - final PipeSubscribeSubscribeResp subscribeResp = - PipeSubscribeSubscribeResp.fromTPipeSubscribeResp(resp); - return subscribeResp.getTopics(); - } - - Map unsubscribe(final Set topicNames) throws SubscriptionException { - final PipeSubscribeUnsubscribeReq req; - try { - req = PipeSubscribeUnsubscribeReq.toTPipeSubscribeReq(topicNames); - } catch (final IOException e) { - LOGGER.warn( - "IOException occurred when SubscriptionProvider {} serialize unsubscribe request {}", - this, - topicNames, - e); - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(req); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} unsubscribe with request {}, set SubscriptionProvider unavailable", - this, - topicNames, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - final PipeSubscribeUnsubscribeResp unsubscribeResp = - PipeSubscribeUnsubscribeResp.fromTPipeSubscribeResp(resp); - return unsubscribeResp.getTopics(); - } - - List poll(final SubscriptionPollRequest pollMessage) - throws SubscriptionException { - final PipeSubscribePollReq req; - try { - req = PipeSubscribePollReq.toTPipeSubscribeReq(pollMessage); - } catch (final IOException e) { - LOGGER.warn( - "IOException occurred when SubscriptionProvider {} serialize poll request {}", - this, - pollMessage, - e); - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(req); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} poll with request {}, set SubscriptionProvider unavailable", - this, - pollMessage, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - final PipeSubscribePollResp pollResp = PipeSubscribePollResp.fromTPipeSubscribeResp(resp); - return pollResp.getResponses(); - } - - void commit(final List subscriptionCommitContexts, final boolean nack) - throws SubscriptionException { - final PipeSubscribeCommitReq req; - try { - req = PipeSubscribeCommitReq.toTPipeSubscribeReq(subscriptionCommitContexts, nack); - } catch (final IOException e) { - LOGGER.warn( - "IOException occurred when SubscriptionProvider {} serialize commit request {}", - this, - subscriptionCommitContexts, - e); - throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); - } - final TPipeSubscribeResp resp; - try { - resp = getSessionConnection().pipeSubscribe(req); - } catch (final TException e) { - // Assume provider unavailable - LOGGER.warn( - "TException occurred when SubscriptionProvider {} commit with request {}, set SubscriptionProvider unavailable", - this, - subscriptionCommitContexts, - e); - setUnavailable(); - throw new SubscriptionConnectionException(e.getMessage(), e); - } - verifyPipeSubscribeSuccess(resp.status); - } - - private static void verifyPipeSubscribeSuccess(final TSStatus status) - throws SubscriptionException { - switch (status.code) { - case 200: // SUCCESS_STATUS - return; - case 1902: // SUBSCRIPTION_HANDSHAKE_ERROR - case 1903: // SUBSCRIPTION_HEARTBEAT_ERROR - case 1904: // SUBSCRIPTION_POLL_ERROR - case 1905: // SUBSCRIPTION_COMMIT_ERROR - case 1906: // SUBSCRIPTION_CLOSE_ERROR - case 1907: // SUBSCRIPTION_SUBSCRIBE_ERROR - case 1908: // SUBSCRIPTION_UNSUBSCRIBE_ERROR - LOGGER.warn( - "Internal error occurred, status code {}, status message {}", - status.code, - status.message); - throw new SubscriptionRuntimeNonCriticalException(status.message); - case 1900: // SUBSCRIPTION_VERSION_ERROR - case 1901: // SUBSCRIPTION_TYPE_ERROR - case 1909: // SUBSCRIPTION_MISSING_CUSTOMER - default: - LOGGER.warn( - "Internal error occurred, status code {}, status message {}", - status.code, - status.message); - throw new SubscriptionRuntimeCriticalException(status.message); - } - } - - @Override - public String toString() { - return "SubscriptionProvider{endPoint=" - + endPoint - + ", dataNodeId=" - + dataNodeId - + ", consumerId=" - + consumerId - + ", consumerGroupId=" - + consumerGroupId - + ", isAvailable=" - + isAvailable - + ", isClosed=" - + isClosed - + "}"; - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProviders.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProviders.java deleted file mode 100644 index 6d250f6d20177..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProviders.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -final class SubscriptionProviders { - - private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionProviders.class); - - private final SortedMap subscriptionProviders = - new ConcurrentSkipListMap<>(); - private int nextDataNodeId = -1; - - private final ReentrantReadWriteLock subscriptionProvidersLock = new ReentrantReadWriteLock(true); - - private final Set initialEndpoints; - - SubscriptionProviders(final Set initialEndpoints) { - this.initialEndpoints = initialEndpoints; - } - - /////////////////////////////// lock /////////////////////////////// - - void acquireReadLock() { - subscriptionProvidersLock.readLock().lock(); - } - - void releaseReadLock() { - subscriptionProvidersLock.readLock().unlock(); - } - - void acquireWriteLock() { - subscriptionProvidersLock.writeLock().lock(); - } - - void releaseWriteLock() { - subscriptionProvidersLock.writeLock().unlock(); - } - - /////////////////////////////// CRUD /////////////////////////////// - - /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ - void openProviders(final SubscriptionConsumer consumer) throws SubscriptionException { - // close stale providers - closeProviders(); - - for (final TEndPoint endPoint : initialEndpoints) { - final SubscriptionProvider defaultProvider; - final int defaultDataNodeId; - - try { - defaultProvider = consumer.constructProviderAndHandshake(endPoint); - } catch (final Exception e) { - LOGGER.warn("Failed to create connection with {}", endPoint, e); - continue; // try next endpoint - } - defaultDataNodeId = defaultProvider.getDataNodeId(); - addProvider(defaultDataNodeId, defaultProvider); - - final Map allEndPoints; - try { - allEndPoints = defaultProvider.getSessionConnection().fetchAllEndPoints(); - } catch (final Exception e) { - LOGGER.warn("Failed to fetch all endpoints from {}, will retry later...", endPoint, e); - break; // retry later - } - - for (final Map.Entry entry : allEndPoints.entrySet()) { - if (defaultDataNodeId == entry.getKey()) { - continue; - } - - final SubscriptionProvider provider; - try { - provider = consumer.constructProviderAndHandshake(entry.getValue()); - } catch (final Exception e) { - LOGGER.warn( - "Failed to create connection with {}, will retry later...", entry.getValue(), e); - continue; // retry later - } - addProvider(entry.getKey(), provider); - } - - break; - } - - if (hasNoAvailableProviders()) { - throw new SubscriptionConnectionException( - String.format( - "Cluster has no available subscription providers to connect with initial endpoints %s", - initialEndpoints)); - } - - nextDataNodeId = subscriptionProviders.firstKey(); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ - void closeProviders() { - for (final SubscriptionProvider provider : getAllProviders()) { - try { - provider.close(); - } catch (final Exception ignored) { - } - } - subscriptionProviders.clear(); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ - void addProvider(final int dataNodeId, final SubscriptionProvider provider) { - // the subscription provider is opened - LOGGER.info("add new subscription provider {}", provider); - subscriptionProviders.put(dataNodeId, provider); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ - void closeAndRemoveProvider(final int dataNodeId) - throws SubscriptionException, IoTDBConnectionException { - if (!containsProvider(dataNodeId)) { - return; - } - final SubscriptionProvider provider = subscriptionProviders.get(dataNodeId); - try { - provider.close(); - } finally { - LOGGER.info("close and remove stale subscription provider {}", provider); - subscriptionProviders.remove(dataNodeId); - } - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - boolean hasNoProviders() { - return subscriptionProviders.isEmpty(); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - List getAllProviders() { - return new ArrayList<>(subscriptionProviders.values()); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - SubscriptionProvider getProvider(final int dataNodeId) { - return containsProvider(dataNodeId) ? subscriptionProviders.get(dataNodeId) : null; - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - boolean hasNoAvailableProviders() { - return subscriptionProviders.values().stream().noneMatch(SubscriptionProvider::isAvailable); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - boolean containsProvider(final int dataNodeId) { - return subscriptionProviders.containsKey(dataNodeId); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - List getAllAvailableProviders() { - return subscriptionProviders.values().stream() - .filter(SubscriptionProvider::isAvailable) - .collect(Collectors.toList()); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - void updateNextDataNodeId() { - final SortedMap subProviders = - subscriptionProviders.tailMap(nextDataNodeId + 1); - nextDataNodeId = - subProviders.isEmpty() ? subscriptionProviders.firstKey() : subProviders.firstKey(); - } - - /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ - SubscriptionProvider getNextAvailableProvider() { - if (hasNoAvailableProviders()) { - return null; - } - - SubscriptionProvider provider; - provider = getProvider(nextDataNodeId); - while (Objects.isNull(provider) || !provider.isAvailable()) { - updateNextDataNodeId(); - provider = getProvider(nextDataNodeId); - } - updateNextDataNodeId(); - - return provider; - } - - /////////////////////////////// heartbeat /////////////////////////////// - - void heartbeat(final SubscriptionConsumer consumer) { - if (consumer.isClosed()) { - return; - } - - acquireWriteLock(); - try { - heartbeatInternal(); - } finally { - releaseWriteLock(); - } - } - - private void heartbeatInternal() { - for (final SubscriptionProvider provider : getAllProviders()) { - try { - provider.heartbeat(); - provider.setAvailable(); - } catch (final Exception e) { - LOGGER.warn( - "something unexpected happened when sending heartbeat to subscription provider {}, set subscription provider unavailable", - provider, - e); - provider.setUnavailable(); - } - } - } - - /////////////////////////////// sync endpoints /////////////////////////////// - - void sync(final SubscriptionConsumer consumer) { - if (consumer.isClosed()) { - return; - } - - acquireWriteLock(); - try { - syncInternal(consumer); - } finally { - releaseWriteLock(); - } - } - - private void syncInternal(final SubscriptionConsumer consumer) { - if (hasNoAvailableProviders()) { - try { - openProviders(consumer); - } catch (final Exception e) { - LOGGER.warn("something unexpected happened when syncing subscription endpoints...", e); - return; - } - } - - final Map allEndPoints; - try { - allEndPoints = consumer.fetchAllEndPointsWithRedirection(); - } catch (final Exception e) { - LOGGER.warn("Failed to fetch all endpoints, will retry later...", e); - return; // retry later - } - - // add new providers or handshake existing providers - for (final Map.Entry entry : allEndPoints.entrySet()) { - final SubscriptionProvider provider = getProvider(entry.getKey()); - if (Objects.isNull(provider)) { - // new provider - final TEndPoint endPoint = entry.getValue(); - final SubscriptionProvider newProvider; - try { - newProvider = consumer.constructProviderAndHandshake(endPoint); - } catch (final Exception e) { - LOGGER.warn( - "Failed to create connection with endpoint {}, will retry later...", endPoint, e); - continue; // retry later - } - addProvider(entry.getKey(), newProvider); - } else { - // existing provider - try { - provider.heartbeat(); - provider.setAvailable(); - } catch (final Exception e) { - LOGGER.warn( - "something unexpected happened when sending heartbeat to subscription provider {}, set subscription provider unavailable", - provider, - e); - provider.setUnavailable(); - } - // close and remove unavailable provider (reset the connection as much as possible) - if (!provider.isAvailable()) { - try { - closeAndRemoveProvider(entry.getKey()); - } catch (final Exception e) { - LOGGER.warn( - "Exception occurred when closing and removing subscription provider with data node id {}", - entry.getKey(), - e); - } - } - } - } - - // close and remove stale providers - for (final SubscriptionProvider provider : getAllProviders()) { - final int dataNodeId = provider.getDataNodeId(); - if (!allEndPoints.containsKey(dataNodeId)) { - try { - closeAndRemoveProvider(dataNodeId); - } catch (final Exception e) { - LOGGER.warn( - "Exception occurred when closing and removing subscription provider with data node id {}", - dataNodeId, - e); - } - } - } - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("SubscriptionProviders{"); - for (final Map.Entry entry : subscriptionProviders.entrySet()) { - sb.append(entry.getValue().toString()).append(", "); - } - if (!subscriptionProviders.isEmpty()) { - sb.delete(sb.length() - 2, sb.length()); - } - sb.append("}"); - return sb.toString(); - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java deleted file mode 100644 index 377341874044c..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer; - -import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; -import org.apache.iotdb.session.subscription.util.IdentifierUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -/** - * The {@link SubscriptionPullConsumer} corresponds to the pull consumption mode in the message - * queue. - * - *

User code needs to actively call the data retrieval logic, i.e., the {@link #poll} method. - * - *

Auto-commit for consumption progress can be configured in {@link #autoCommit}. - * - *

NOTE: It is not recommended to use the {@link #poll} method with the same consumer in a - * multithreaded environment. Instead, it is advised to increase the number of consumers to improve - * data retrieval parallelism. - */ -public class SubscriptionPullConsumer extends SubscriptionConsumer { - - private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionPullConsumer.class); - - private final boolean autoCommit; - private final long autoCommitIntervalMs; - - private SortedMap> uncommittedMessages; - - private final AtomicBoolean isClosed = new AtomicBoolean(true); - - @Override - boolean isClosed() { - return isClosed.get(); - } - - /////////////////////////////// ctor /////////////////////////////// - - protected SubscriptionPullConsumer(final SubscriptionPullConsumer.Builder builder) { - super(builder); - - this.autoCommit = builder.autoCommit; - this.autoCommitIntervalMs = builder.autoCommitIntervalMs; - } - - public SubscriptionPullConsumer(final Properties properties) { - this( - properties, - (Boolean) - properties.getOrDefault( - ConsumerConstant.AUTO_COMMIT_KEY, ConsumerConstant.AUTO_COMMIT_DEFAULT_VALUE), - (Long) - properties.getOrDefault( - ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_KEY, - ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_DEFAULT_VALUE)); - } - - private SubscriptionPullConsumer( - final Properties properties, final boolean autoCommit, final long autoCommitIntervalMs) { - super( - new Builder().autoCommit(autoCommit).autoCommitIntervalMs(autoCommitIntervalMs), - properties); - - this.autoCommit = autoCommit; - this.autoCommitIntervalMs = - Math.max(autoCommitIntervalMs, ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_MIN_VALUE); - } - - /////////////////////////////// open & close /////////////////////////////// - - public synchronized void open() throws SubscriptionException { - if (!isClosed.get()) { - return; - } - - super.open(); - - // set isClosed to false before submitting workers - isClosed.set(false); - - // submit auto poll worker if enabling auto commit - if (autoCommit) { - uncommittedMessages = new ConcurrentSkipListMap<>(); - submitAutoCommitWorker(); - } - } - - @Override - public synchronized void close() { - if (isClosed.get()) { - return; - } - - if (autoCommit) { - // commit all uncommitted messages - commitAllUncommittedMessages(); - } - - super.close(); - isClosed.set(true); - } - - /////////////////////////////// poll & commit /////////////////////////////// - - public List poll(final Duration timeout) throws SubscriptionException { - return poll(Collections.emptySet(), timeout.toMillis()); - } - - public List poll(final long timeoutMs) throws SubscriptionException { - return poll(Collections.emptySet(), timeoutMs); - } - - public List poll(final Set topicNames, final Duration timeout) - throws SubscriptionException { - return poll(topicNames, timeout.toMillis()); - } - - @Override - public List poll(final Set topicNames, final long timeoutMs) - throws SubscriptionException { - // parse topic names from external source - final Set parsedTopicNames = - topicNames.stream().map(IdentifierUtils::parseIdentifier).collect(Collectors.toSet()); - - // poll messages - final List messages = super.poll(parsedTopicNames, timeoutMs); - - // add to uncommitted messages - if (autoCommit) { - final long currentTimestamp = System.currentTimeMillis(); - long index = currentTimestamp / autoCommitIntervalMs; - if (currentTimestamp % autoCommitIntervalMs == 0) { - index -= 1; - } - uncommittedMessages - .computeIfAbsent(index, o -> new ConcurrentSkipListSet<>()) - .addAll(messages); - } - - return messages; - } - - /////////////////////////////// commit /////////////////////////////// - - public void commitSync(final SubscriptionMessage message) throws SubscriptionException { - super.ack(Collections.singletonList(message)); - } - - public void commitSync(final Iterable messages) - throws SubscriptionException { - super.ack(messages); - } - - public CompletableFuture commitAsync(final SubscriptionMessage message) { - return super.commitAsync(Collections.singletonList(message)); - } - - public CompletableFuture commitAsync(final Iterable messages) { - return super.commitAsync(messages); - } - - public void commitAsync(final SubscriptionMessage message, final AsyncCommitCallback callback) { - super.commitAsync(Collections.singletonList(message), callback); - } - - public void commitAsync( - final Iterable messages, final AsyncCommitCallback callback) { - super.commitAsync(messages, callback); - } - - /////////////////////////////// auto commit /////////////////////////////// - - private void submitAutoCommitWorker() { - final ScheduledFuture[] future = new ScheduledFuture[1]; - future[0] = - SubscriptionExecutorServiceManager.submitAutoCommitWorker( - () -> { - if (isClosed()) { - if (Objects.nonNull(future[0])) { - future[0].cancel(false); - LOGGER.info("SubscriptionPullConsumer {} cancel auto commit worker", this); - } - return; - } - new AutoCommitWorker().run(); - }, - autoCommitIntervalMs); - LOGGER.info("SubscriptionPullConsumer {} submit auto commit worker", this); - } - - private class AutoCommitWorker implements Runnable { - @Override - public void run() { - if (isClosed()) { - return; - } - - final long currentTimestamp = System.currentTimeMillis(); - long index = currentTimestamp / autoCommitIntervalMs; - if (currentTimestamp % autoCommitIntervalMs == 0) { - index -= 1; - } - - for (final Map.Entry> entry : - uncommittedMessages.headMap(index).entrySet()) { - try { - ack(entry.getValue()); - uncommittedMessages.remove(entry.getKey()); - } catch (final Exception e) { - LOGGER.warn("something unexpected happened when auto commit messages...", e); - } - } - } - } - - private void commitAllUncommittedMessages() { - for (final Map.Entry> entry : uncommittedMessages.entrySet()) { - try { - ack(entry.getValue()); - uncommittedMessages.remove(entry.getKey()); - } catch (final Exception e) { - LOGGER.warn("something unexpected happened when commit messages during close", e); - } - } - } - - /////////////////////////////// builder /////////////////////////////// - - public static class Builder extends SubscriptionConsumer.Builder { - - private boolean autoCommit = ConsumerConstant.AUTO_COMMIT_DEFAULT_VALUE; - private long autoCommitIntervalMs = ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_DEFAULT_VALUE; - - @Override - public Builder host(final String host) { - super.host(host); - return this; - } - - @Override - public Builder port(final int port) { - super.port(port); - return this; - } - - @Override - public Builder nodeUrls(final List nodeUrls) { - super.nodeUrls(nodeUrls); - return this; - } - - @Override - public Builder username(final String username) { - super.username(username); - return this; - } - - @Override - public Builder password(final String password) { - super.password(password); - return this; - } - - @Override - public Builder consumerId(final String consumerId) { - super.consumerId(consumerId); - return this; - } - - @Override - public Builder consumerGroupId(final String consumerGroupId) { - super.consumerGroupId(consumerGroupId); - return this; - } - - @Override - public Builder heartbeatIntervalMs(final long heartbeatIntervalMs) { - super.heartbeatIntervalMs(heartbeatIntervalMs); - return this; - } - - @Override - public Builder endpointsSyncIntervalMs(final long endpointsSyncIntervalMs) { - super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); - return this; - } - - @Override - public Builder fileSaveDir(final String fileSaveDir) { - super.fileSaveDir(fileSaveDir); - return this; - } - - @Override - public Builder fileSaveFsync(final boolean fileSaveFsync) { - super.fileSaveFsync(fileSaveFsync); - return this; - } - - public Builder autoCommit(final boolean autoCommit) { - this.autoCommit = autoCommit; - return this; - } - - public Builder autoCommitIntervalMs(final long autoCommitIntervalMs) { - this.autoCommitIntervalMs = - Math.max(autoCommitIntervalMs, ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_MIN_VALUE); - return this; - } - - @Override - public SubscriptionPullConsumer buildPullConsumer() { - return new SubscriptionPullConsumer(this); - } - - @Override - public SubscriptionPushConsumer buildPushConsumer() { - throw new SubscriptionException( - "SubscriptionPullConsumer.Builder do not support build push consumer."); - } - } - - /////////////////////////////// stringify /////////////////////////////// - - @Override - public String toString() { - return "SubscriptionPullConsumer" + this.coreReportMessage(); - } - - @Override - protected Map coreReportMessage() { - final Map coreReportMessage = super.coreReportMessage(); - coreReportMessage.put("autoCommit", String.valueOf(autoCommit)); - return coreReportMessage; - } - - @Override - protected Map allReportMessage() { - final Map allReportMessage = super.allReportMessage(); - allReportMessage.put("autoCommit", String.valueOf(autoCommit)); - allReportMessage.put("autoCommitIntervalMs", String.valueOf(autoCommitIntervalMs)); - if (autoCommit) { - allReportMessage.put("uncommittedMessages", uncommittedMessages.toString()); - } - return allReportMessage; - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java deleted file mode 100644 index e2dfe7974575d..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer; - -import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * The {@link SubscriptionPushConsumer} corresponds to the push consumption mode in the message - * queue. - * - *

User code is triggered by newly arrived data events and only needs to pre-configure message - * acknowledgment strategy ({@link #ackStrategy}) and consumption handling logic ({@link - * #consumeListener}). - * - *

User code does not need to manually commit the consumption progress. - */ -public class SubscriptionPushConsumer extends SubscriptionConsumer { - - private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionPushConsumer.class); - - private final AckStrategy ackStrategy; - private final ConsumeListener consumeListener; - - // avoid interval less than or equal to zero - private final long autoPollIntervalMs; - private final long autoPollTimeoutMs; - - private final AtomicBoolean isClosed = new AtomicBoolean(true); - - protected SubscriptionPushConsumer(final Builder builder) { - super(builder); - - this.ackStrategy = builder.ackStrategy; - this.consumeListener = builder.consumeListener; - - this.autoPollIntervalMs = builder.autoPollIntervalMs; - this.autoPollTimeoutMs = builder.autoPollTimeoutMs; - } - - public SubscriptionPushConsumer(final Properties config) { - this( - config, - (AckStrategy) - config.getOrDefault(ConsumerConstant.ACK_STRATEGY_KEY, AckStrategy.defaultValue()), - (ConsumeListener) - config.getOrDefault( - ConsumerConstant.CONSUME_LISTENER_KEY, - (ConsumeListener) message -> ConsumeResult.SUCCESS), - (Long) - config.getOrDefault( - ConsumerConstant.AUTO_POLL_INTERVAL_MS_KEY, - ConsumerConstant.AUTO_POLL_INTERVAL_MS_DEFAULT_VALUE), - (Long) - config.getOrDefault( - ConsumerConstant.AUTO_POLL_TIMEOUT_MS_KEY, - ConsumerConstant.AUTO_POLL_TIMEOUT_MS_DEFAULT_VALUE)); - } - - private SubscriptionPushConsumer( - final Properties config, - final AckStrategy ackStrategy, - final ConsumeListener consumeListener, - final long autoPollIntervalMs, - final long autoPollTimeoutMs) { - super( - new Builder() - .ackStrategy(ackStrategy) - .consumeListener(consumeListener) - .autoPollIntervalMs(autoPollIntervalMs) - .autoPollTimeoutMs(autoPollTimeoutMs), - config); - - this.ackStrategy = ackStrategy; - this.consumeListener = consumeListener; - - this.autoPollIntervalMs = Math.max(autoPollIntervalMs, 1); - this.autoPollTimeoutMs = - Math.max(autoPollTimeoutMs, ConsumerConstant.AUTO_POLL_TIMEOUT_MS_MIN_VALUE); - } - - /////////////////////////////// open & close /////////////////////////////// - - public synchronized void open() throws SubscriptionException { - if (!isClosed.get()) { - return; - } - - super.open(); - - // set isClosed to false before submitting workers - isClosed.set(false); - - // submit auto poll worker - submitAutoPollWorker(); - } - - @Override - public synchronized void close() { - if (isClosed.get()) { - return; - } - - super.close(); - isClosed.set(true); - } - - @Override - boolean isClosed() { - return isClosed.get(); - } - - /////////////////////////////// auto poll /////////////////////////////// - - private void submitAutoPollWorker() { - final ScheduledFuture[] future = new ScheduledFuture[1]; - future[0] = - SubscriptionExecutorServiceManager.submitAutoPollWorker( - () -> { - if (isClosed()) { - if (Objects.nonNull(future[0])) { - future[0].cancel(false); - LOGGER.info("SubscriptionPushConsumer {} cancel auto poll worker", this); - } - return; - } - new AutoPollWorker().run(); - }, - autoPollIntervalMs); - LOGGER.info("SubscriptionPushConsumer {} submit auto poll worker", this); - } - - class AutoPollWorker implements Runnable { - @Override - public void run() { - if (isClosed()) { - return; - } - - if (subscribedTopics.isEmpty()) { - return; - } - - try { - final List messages = - poll(subscribedTopics.keySet(), autoPollTimeoutMs); - - if (ackStrategy.equals(AckStrategy.BEFORE_CONSUME)) { - ack(messages); - } - - final List messagesToAck = new ArrayList<>(); - final List messagesToNack = new ArrayList<>(); - for (final SubscriptionMessage message : messages) { - final ConsumeResult consumeResult; - try { - consumeResult = consumeListener.onReceive(message); - if (Objects.equals(consumeResult, ConsumeResult.SUCCESS)) { - messagesToAck.add(message); - } else { - LOGGER.warn("Consumer listener result failure when consuming message: {}", message); - messagesToNack.add(message); - } - } catch (final Exception e) { - LOGGER.warn( - "Consumer listener raised an exception while consuming message: {}", message, e); - messagesToNack.add(message); - } - } - - if (ackStrategy.equals(AckStrategy.AFTER_CONSUME)) { - ack(messagesToAck); - nack(messagesToNack); - } - } catch (final Exception e) { - LOGGER.warn("something unexpected happened when auto poll messages...", e); - } - } - } - - /////////////////////////////// builder /////////////////////////////// - - public static class Builder extends SubscriptionConsumer.Builder { - - private AckStrategy ackStrategy = AckStrategy.defaultValue(); - private ConsumeListener consumeListener = message -> ConsumeResult.SUCCESS; - - private long autoPollIntervalMs = ConsumerConstant.AUTO_POLL_INTERVAL_MS_DEFAULT_VALUE; - private long autoPollTimeoutMs = ConsumerConstant.AUTO_POLL_TIMEOUT_MS_DEFAULT_VALUE; - - @Override - public Builder host(final String host) { - super.host(host); - return this; - } - - @Override - public Builder port(final int port) { - super.port(port); - return this; - } - - @Override - public Builder nodeUrls(final List nodeUrls) { - super.nodeUrls(nodeUrls); - return this; - } - - @Override - public Builder username(final String username) { - super.username(username); - return this; - } - - @Override - public Builder password(final String password) { - super.password(password); - return this; - } - - @Override - public Builder consumerId(final String consumerId) { - super.consumerId(consumerId); - return this; - } - - @Override - public Builder consumerGroupId(final String consumerGroupId) { - super.consumerGroupId(consumerGroupId); - return this; - } - - @Override - public Builder heartbeatIntervalMs(final long heartbeatIntervalMs) { - super.heartbeatIntervalMs(heartbeatIntervalMs); - return this; - } - - @Override - public Builder endpointsSyncIntervalMs(final long endpointsSyncIntervalMs) { - super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); - return this; - } - - @Override - public Builder fileSaveDir(final String fileSaveDir) { - super.fileSaveDir(fileSaveDir); - return this; - } - - @Override - public Builder fileSaveFsync(final boolean fileSaveFsync) { - super.fileSaveFsync(fileSaveFsync); - return this; - } - - public Builder ackStrategy(final AckStrategy ackStrategy) { - this.ackStrategy = ackStrategy; - return this; - } - - public Builder consumeListener(final ConsumeListener consumeListener) { - this.consumeListener = consumeListener; - return this; - } - - public Builder autoPollIntervalMs(final long autoPollIntervalMs) { - // avoid interval less than or equal to zero - this.autoPollIntervalMs = Math.max(autoPollIntervalMs, 1); - return this; - } - - public Builder autoPollTimeoutMs(final long autoPollTimeoutMs) { - this.autoPollTimeoutMs = - Math.max(autoPollTimeoutMs, ConsumerConstant.AUTO_POLL_TIMEOUT_MS_MIN_VALUE); - return this; - } - - @Override - public SubscriptionPullConsumer buildPullConsumer() { - throw new SubscriptionException( - "SubscriptionPushConsumer.Builder do not support build pull consumer."); - } - - @Override - public SubscriptionPushConsumer buildPushConsumer() { - return new SubscriptionPushConsumer(this); - } - } - - /////////////////////////////// stringify /////////////////////////////// - - @Override - public String toString() { - return "SubscriptionPushConsumer" + this.coreReportMessage(); - } - - @Override - protected Map coreReportMessage() { - final Map coreReportMessage = super.coreReportMessage(); - coreReportMessage.put("ackStrategy", ackStrategy.toString()); - return coreReportMessage; - } - - @Override - protected Map allReportMessage() { - final Map allReportMessage = super.allReportMessage(); - allReportMessage.put("ackStrategy", ackStrategy.toString()); - allReportMessage.put("autoPollIntervalMs", String.valueOf(autoPollIntervalMs)); - allReportMessage.put("autoPollTimeoutMs", String.valueOf(autoPollTimeoutMs)); - return allReportMessage; - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java new file mode 100644 index 0000000000000..a12340e9d7662 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -0,0 +1,1435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionPipeTimeoutException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionPollTimeoutException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionTimeoutException; +import org.apache.iotdb.rpc.subscription.payload.poll.ErrorPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.FilePiecePayload; +import org.apache.iotdb.rpc.subscription.payload.poll.FileSealPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; +import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; +import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.util.CollectionUtils; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; +import org.apache.iotdb.session.subscription.util.PollTimer; +import org.apache.iotdb.session.subscription.util.RandomStringGenerator; +import org.apache.iotdb.session.util.SessionUtils; + +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URLEncoder; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.ERROR; +import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.FILE_INIT; +import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.TABLETS; +import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.TERMINATION; +import static org.apache.iotdb.session.subscription.util.SetPartitioner.partition; + +abstract class AbstractSubscriptionConsumer implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSubscriptionConsumer.class); + + private static final long SLEEP_MS = 100L; + private static final long SLEEP_DELTA_MS = 50L; + private static final long TIMER_DELTA_MS = 250L; + + private final String username; + private final String password; + + protected String consumerId; + protected String consumerGroupId; + + private final long heartbeatIntervalMs; + private final long endpointsSyncIntervalMs; + + private final AbstractSubscriptionProviders providers; + + private final AtomicBoolean isClosed = new AtomicBoolean(true); + // This variable indicates whether the consumer has ever been closed. + private final AtomicBoolean isReleased = new AtomicBoolean(false); + + private final String fileSaveDir; + private final boolean fileSaveFsync; + private final Set inFlightFilesCommitContextSet = new HashSet<>(); + + private final int thriftMaxFrameSize; + private final int maxPollParallelism; + + @SuppressWarnings("java:S3077") + protected volatile Map subscribedTopics = new HashMap<>(); + + @Deprecated + public boolean allSnapshotTopicMessagesHaveBeenConsumed() { + return allTopicMessagesHaveBeenConsumed(subscribedTopics.keySet()); + } + + public boolean allTopicMessagesHaveBeenConsumed() { + return allTopicMessagesHaveBeenConsumed(subscribedTopics.keySet()); + } + + private boolean allTopicMessagesHaveBeenConsumed(final Collection topicNames) { + // For the topic that needs to be detected, there are two scenarios to consider: + // 1. If configs as live, it cannot be determined whether the topic has been fully consumed. + // 2. If configs as snapshot, it means the topic has not been automatically unsubscribed. + // Therefore, the logic can be summarized as follows: if there is a matching topic in subscribed + // topics, then it has not been fully consumed. + return topicNames.stream().map(subscribedTopics::get).noneMatch(Objects::nonNull); + } + + /////////////////////////////// getter /////////////////////////////// + + public String getConsumerId() { + return consumerId; + } + + public String getConsumerGroupId() { + return consumerGroupId; + } + + /////////////////////////////// ctor /////////////////////////////// + + protected AbstractSubscriptionConsumer(final AbstractSubscriptionConsumerBuilder builder) { + final Set initialEndpoints = new HashSet<>(); + // From org.apache.iotdb.session.Session.getNodeUrls + // Priority is given to `host:port` over `nodeUrls`. + if (Objects.nonNull(builder.host) || Objects.nonNull(builder.port)) { + if (Objects.isNull(builder.host)) { + builder.host = SessionConfig.DEFAULT_HOST; + } + if (Objects.isNull(builder.port)) { + builder.port = SessionConfig.DEFAULT_PORT; + } + initialEndpoints.add(new TEndPoint(builder.host, builder.port)); + } else if (Objects.isNull(builder.nodeUrls)) { + builder.host = SessionConfig.DEFAULT_HOST; + builder.port = SessionConfig.DEFAULT_PORT; + initialEndpoints.add(new TEndPoint(builder.host, builder.port)); + } else { + initialEndpoints.addAll(SessionUtils.parseSeedNodeUrls(builder.nodeUrls)); + } + this.providers = new AbstractSubscriptionProviders(initialEndpoints); + + this.username = builder.username; + this.password = builder.password; + + this.consumerId = builder.consumerId; + this.consumerGroupId = builder.consumerGroupId; + + this.heartbeatIntervalMs = builder.heartbeatIntervalMs; + this.endpointsSyncIntervalMs = builder.endpointsSyncIntervalMs; + + this.fileSaveDir = builder.fileSaveDir; + this.fileSaveFsync = builder.fileSaveFsync; + + this.thriftMaxFrameSize = builder.thriftMaxFrameSize; + this.maxPollParallelism = builder.maxPollParallelism; + } + + protected AbstractSubscriptionConsumer( + final AbstractSubscriptionConsumerBuilder builder, final Properties properties) { + this( + builder + .host((String) properties.get(ConsumerConstant.HOST_KEY)) + .port((Integer) properties.get(ConsumerConstant.PORT_KEY)) + .nodeUrls((List) properties.get(ConsumerConstant.NODE_URLS_KEY)) + .username( + (String) + properties.getOrDefault( + ConsumerConstant.USERNAME_KEY, SessionConfig.DEFAULT_USER)) + .password( + (String) + properties.getOrDefault( + ConsumerConstant.PASSWORD_KEY, SessionConfig.DEFAULT_PASSWORD)) + .consumerId((String) properties.get(ConsumerConstant.CONSUMER_ID_KEY)) + .consumerGroupId((String) properties.get(ConsumerConstant.CONSUMER_GROUP_ID_KEY)) + .heartbeatIntervalMs( + (Long) + properties.getOrDefault( + ConsumerConstant.HEARTBEAT_INTERVAL_MS_KEY, + ConsumerConstant.HEARTBEAT_INTERVAL_MS_DEFAULT_VALUE)) + .endpointsSyncIntervalMs( + (Long) + properties.getOrDefault( + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_KEY, + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_DEFAULT_VALUE)) + .fileSaveDir( + (String) + properties.getOrDefault( + ConsumerConstant.FILE_SAVE_DIR_KEY, + ConsumerConstant.FILE_SAVE_DIR_DEFAULT_VALUE)) + .fileSaveFsync( + (Boolean) + properties.getOrDefault( + ConsumerConstant.FILE_SAVE_FSYNC_KEY, + ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE)) + .thriftMaxFrameSize( + (Integer) + properties.getOrDefault( + ConsumerConstant.THRIFT_MAX_FRAME_SIZE_KEY, + SessionConfig.DEFAULT_MAX_FRAME_SIZE)) + .maxPollParallelism( + (Integer) + properties.getOrDefault( + ConsumerConstant.MAX_POLL_PARALLELISM_KEY, + ConsumerConstant.MAX_POLL_PARALLELISM_DEFAULT_VALUE))); + } + + /////////////////////////////// open & close /////////////////////////////// + + private void checkIfHasBeenClosed() throws SubscriptionException { + if (isReleased.get()) { + final String errorMessage = + String.format("%s has ever been closed, unsupported operation after closing.", this); + LOGGER.error(errorMessage); + throw new SubscriptionException(errorMessage); + } + } + + private void checkIfOpened() throws SubscriptionException { + if (isClosed.get()) { + final String errorMessage = + String.format("%s is not yet open, please open the subscription consumer first.", this); + LOGGER.error(errorMessage); + throw new SubscriptionException(errorMessage); + } + } + + protected synchronized void open() throws SubscriptionException { + checkIfHasBeenClosed(); + + if (!isClosed.get()) { + return; + } + + // open subscription providers + providers.acquireWriteLock(); + try { + providers.openProviders(this); // throw SubscriptionException + } finally { + providers.releaseWriteLock(); + } + + // set isClosed to false before submitting workers + isClosed.set(false); + + // submit heartbeat worker + submitHeartbeatWorker(); + + // submit endpoints syncer + submitEndpointsSyncer(); + } + + @Override + public synchronized void close() { + if (isClosed.get()) { + return; + } + + // close subscription providers + providers.acquireWriteLock(); + try { + providers.closeProviders(); + } finally { + providers.releaseWriteLock(); + } + + isClosed.set(true); + + // mark is released to avoid reopening after closing + isReleased.set(true); + } + + boolean isClosed() { + return isClosed.get(); + } + + /////////////////////////////// subscribe & unsubscribe /////////////////////////////// + + protected void subscribe(final String topicName) throws SubscriptionException { + subscribe(Collections.singleton(topicName)); + } + + protected void subscribe(final String... topicNames) throws SubscriptionException { + subscribe(new HashSet<>(Arrays.asList(topicNames))); + } + + protected void subscribe(final Set topicNames) throws SubscriptionException { + // parse topic names from external source + subscribe(topicNames, true); + } + + private void subscribe(Set topicNames, final boolean needParse) + throws SubscriptionException { + checkIfOpened(); + + if (needParse) { + topicNames = + topicNames.stream() + .map(IdentifierUtils::checkAndParseIdentifier) + .collect(Collectors.toSet()); + } + + providers.acquireReadLock(); + try { + subscribeWithRedirection(topicNames); + } finally { + providers.releaseReadLock(); + } + } + + protected void unsubscribe(final String topicName) throws SubscriptionException { + unsubscribe(Collections.singleton(topicName)); + } + + protected void unsubscribe(final String... topicNames) throws SubscriptionException { + unsubscribe(new HashSet<>(Arrays.asList(topicNames))); + } + + protected void unsubscribe(final Set topicNames) throws SubscriptionException { + // parse topic names from external source + unsubscribe(topicNames, true); + } + + private void unsubscribe(Set topicNames, final boolean needParse) + throws SubscriptionException { + checkIfOpened(); + + if (needParse) { + topicNames = + topicNames.stream() + .map(IdentifierUtils::checkAndParseIdentifier) + .collect(Collectors.toSet()); + } + + providers.acquireReadLock(); + try { + unsubscribeWithRedirection(topicNames); + } finally { + providers.releaseReadLock(); + } + } + + /////////////////////////////// subscription provider /////////////////////////////// + + protected abstract AbstractSubscriptionProvider constructSubscriptionProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize); + + AbstractSubscriptionProvider constructProviderAndHandshake(final TEndPoint endPoint) + throws SubscriptionException { + final AbstractSubscriptionProvider provider = + constructSubscriptionProvider( + endPoint, + this.username, + this.password, + this.consumerId, + this.consumerGroupId, + this.thriftMaxFrameSize); + try { + provider.handshake(); + } catch (final Exception e) { + try { + provider.close(); + } catch (final Exception ignored) { + } + throw new SubscriptionConnectionException( + String.format( + "Failed to handshake with subscription provider %s because of %s", provider, e), + e); + } + + // update consumer id and consumer group id if not exist + if (Objects.isNull(this.consumerId)) { + this.consumerId = provider.getConsumerId(); + } + if (Objects.isNull(this.consumerGroupId)) { + this.consumerGroupId = provider.getConsumerGroupId(); + } + + return provider; + } + + /////////////////////////////// file ops /////////////////////////////// + + private Path getFileDir(final String topicName) throws IOException { + final Path dirPath = + Paths.get(fileSaveDir).resolve(consumerGroupId).resolve(consumerId).resolve(topicName); + Files.createDirectories(dirPath); + return dirPath; + } + + private Path getFilePath( + final SubscriptionCommitContext commitContext, + final String topicName, + final String fileName, + final boolean allowFileAlreadyExistsException, + final boolean allowInvalidPathException) + throws SubscriptionException { + try { + final Path filePath; + try { + filePath = getFileDir(topicName).resolve(fileName); + } catch (final InvalidPathException invalidPathException) { + if (allowInvalidPathException) { + return getFilePath(commitContext, URLEncoder.encode(topicName), fileName, true, false); + } + throw new SubscriptionRuntimeNonCriticalException( + invalidPathException.getMessage(), invalidPathException); + } + + try { + Files.createFile(filePath); + return filePath; + } catch (final FileAlreadyExistsException fileAlreadyExistsException) { + if (allowFileAlreadyExistsException) { + if (inFlightFilesCommitContextSet.contains(commitContext)) { + LOGGER.info( + "Detect already existed file {} when polling topic {}, resume consumption", + fileName, + topicName); + return filePath; + } + final String suffix = RandomStringGenerator.generate(16); + LOGGER.warn( + "Detect already existed file {} when polling topic {}, add random suffix {} to filename", + fileName, + topicName, + suffix); + return getFilePath(commitContext, topicName, fileName + "." + suffix, false, true); + } + throw new SubscriptionRuntimeNonCriticalException( + fileAlreadyExistsException.getMessage(), fileAlreadyExistsException); + } + } catch (final IOException e) { + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + } + + /////////////////////////////// poll /////////////////////////////// + + private final Map< + SubscriptionPollResponseType, + BiFunction>> + responseTransformer = + Collections.unmodifiableMap( + new HashMap< + SubscriptionPollResponseType, + BiFunction< + SubscriptionPollResponse, PollTimer, Optional>>() { + { + put(TABLETS, (resp, timer) -> pollTablets(resp, timer)); + put(FILE_INIT, (resp, timer) -> pollFile(resp, timer)); + put( + ERROR, + (resp, timer) -> { + final ErrorPayload payload = (ErrorPayload) resp.getPayload(); + final String errorMessage = payload.getErrorMessage(); + if (payload.isCritical()) { + throw new SubscriptionRuntimeCriticalException(errorMessage); + } else { + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + }); + put( + TERMINATION, + (resp, timer) -> { + final SubscriptionCommitContext commitContext = resp.getCommitContext(); + final String topicNameToUnsubscribe = commitContext.getTopicName(); + LOGGER.info( + "Termination occurred when SubscriptionConsumer {} polling topics, unsubscribe topic {} automatically", + coreReportMessage(), + topicNameToUnsubscribe); + unsubscribe(Collections.singleton(topicNameToUnsubscribe), false); + return Optional.empty(); + }); + } + }); + + protected List multiplePoll( + /* @NotNull */ final Set topicNames, final long timeoutMs) { + if (topicNames.isEmpty()) { + return Collections.emptyList(); + } + + // execute single task in current thread + final int availableCount = + SubscriptionExecutorServiceManager.getAvailableThreadCountForPollTasks(); + if (availableCount == 0) { + // non-strict timeout + return singlePoll(topicNames, timeoutMs); + } + + // dividing topics + final List tasks = new ArrayList<>(); + final List> partitionedTopicNames = + partition(topicNames, Math.min(maxPollParallelism, availableCount)); + for (final Set partition : partitionedTopicNames) { + tasks.add(new PollTask(partition, timeoutMs)); + } + + // submit multiple tasks to poll messages + final List messages = new ArrayList<>(); + SubscriptionRuntimeCriticalException lastSubscriptionRuntimeCriticalException = null; + try { + // strict timeout + for (final Future> future : + SubscriptionExecutorServiceManager.submitMultiplePollTasks(tasks, timeoutMs)) { + try { + if (future.isCancelled()) { + continue; + } + messages.addAll(future.get()); + } catch (final CancellationException ignored) { + + } catch (final ExecutionException e) { + final Throwable cause = e.getCause(); + if (cause instanceof SubscriptionRuntimeCriticalException) { + final SubscriptionRuntimeCriticalException ex = + (SubscriptionRuntimeCriticalException) cause; + LOGGER.warn( + "SubscriptionRuntimeCriticalException occurred when SubscriptionConsumer {} polling topics {}", + this, + topicNames, + ex); + lastSubscriptionRuntimeCriticalException = ex; + } else { + LOGGER.warn( + "ExecutionException occurred when SubscriptionConsumer {} polling topics {}", + this, + topicNames, + e); + } + } + } + } catch (final InterruptedException e) { + LOGGER.warn( + "InterruptedException occurred when SubscriptionConsumer {} polling topics {}", + this, + topicNames, + e); + Thread.currentThread().interrupt(); // restore interrupted state + } + + // TODO: ignore possible interrupted state? + + // even if a SubscriptionRuntimeCriticalException is encountered, try to deliver the message to + // the client + if (messages.isEmpty() && Objects.nonNull(lastSubscriptionRuntimeCriticalException)) { + throw lastSubscriptionRuntimeCriticalException; + } + + return messages; + } + + private class PollTask implements Callable> { + + private final Set topicNames; + private final long timeoutMs; + + public PollTask(final Set topicNames, final long timeoutMs) { + this.topicNames = topicNames; + this.timeoutMs = timeoutMs; + } + + @Override + public List call() { + return singlePoll(topicNames, timeoutMs); + } + } + + private List singlePoll( + /* @NotNull */ final Set topicNames, final long timeoutMs) + throws SubscriptionException { + if (topicNames.isEmpty()) { + return Collections.emptyList(); + } + + final List messages = new ArrayList<>(); + List currentResponses = new ArrayList<>(); + final PollTimer timer = new PollTimer(System.currentTimeMillis(), timeoutMs); + + try { + do { + final List currentMessages = new ArrayList<>(); + try { + currentResponses.clear(); + currentResponses = pollInternal(topicNames, timer.remainingMs()); + for (final SubscriptionPollResponse response : currentResponses) { + final short responseType = response.getResponseType(); + if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) { + LOGGER.warn("unexpected response type: {}", responseType); + continue; + } + try { + responseTransformer + .getOrDefault( + SubscriptionPollResponseType.valueOf(responseType), + (resp, ignored) -> { + LOGGER.warn("unexpected response type: {}", responseType); + return Optional.empty(); + }) + // TODO: reuse previous timer? + .apply(response, new PollTimer(System.currentTimeMillis(), timeoutMs)) + .ifPresent(currentMessages::add); + } catch (final SubscriptionRuntimeNonCriticalException e) { + LOGGER.warn( + "SubscriptionRuntimeNonCriticalException occurred when SubscriptionConsumer {} polling topics {}", + this, + topicNames, + e); + // assume the corresponding response has been nacked + } + } + } catch (final SubscriptionRuntimeCriticalException e) { + LOGGER.warn( + "SubscriptionRuntimeCriticalException occurred when SubscriptionConsumer {} polling topics {}", + this, + topicNames, + e); + // nack and clear current responses + try { + nack(currentResponses); + currentResponses.clear(); + } catch (final Exception ignored) { + } + // nack and clear result messages + try { + nack(messages); + messages.clear(); + } catch (final Exception ignored) { + } + + // the upper layer perceives ExecutionException + throw e; + } + + // add all current messages to result messages + messages.addAll(currentMessages); + + // TODO: maybe we can poll a few more times + if (!messages.isEmpty()) { + break; + } + + // check if all topic messages have been consumed + if (allTopicMessagesHaveBeenConsumed(topicNames)) { + break; + } + + // update timer + timer.update(); + + // TODO: associated with timeoutMs instead of hardcoding + // random sleep time within the range [SLEEP_DELTA_MS, SLEEP_DELTA_MS + SLEEP_MS) + Thread.sleep(((long) (Math.random() * SLEEP_MS)) + SLEEP_DELTA_MS); + + // the use of TIMER_DELTA_MS here slightly reduces the timeout to avoid being interrupted as + // much as possible + } while (timer.notExpired(TIMER_DELTA_MS)); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // restore interrupted state + } + + if (Thread.currentThread().isInterrupted()) { + // nack and clear current responses + try { + nack(currentResponses); + currentResponses.clear(); + } catch (final Exception ignored) { + } + // nack and clear result messages + try { + nack(messages); + messages.clear(); + } catch (final Exception ignored) { + } + + // the upper layer perceives CancellationException + return Collections.emptyList(); + } + + return messages; + } + + private Optional pollFile( + final SubscriptionPollResponse response, final PollTimer timer) throws SubscriptionException { + final SubscriptionCommitContext commitContext = response.getCommitContext(); + final String fileName = ((FileInitPayload) response.getPayload()).getFileName(); + final String topicName = commitContext.getTopicName(); + final Path filePath = getFilePath(commitContext, topicName, fileName, true, true); + final File file = filePath.toFile(); + try (final RandomAccessFile fileWriter = new RandomAccessFile(file, "rw")) { + return pollFileInternal(commitContext, fileName, file, fileWriter, timer); + } catch (final Exception e) { + if (!(e instanceof SubscriptionPollTimeoutException)) { + inFlightFilesCommitContextSet.remove(commitContext); + } + // construct temporary message to nack + nack( + Collections.singletonList( + new SubscriptionMessage(commitContext, file.getAbsolutePath(), null))); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + } + + private Optional pollFileInternal( + final SubscriptionCommitContext commitContext, + final String rawFileName, + final File file, + final RandomAccessFile fileWriter, + final PollTimer timer) + throws IOException, SubscriptionException { + long writingOffset = fileWriter.length(); + + LOGGER.info( + "{} start to poll file {} with commit context {} at offset {}", + this, + file.getAbsolutePath(), + commitContext, + writingOffset); + + fileWriter.seek(writingOffset); + while (true) { + timer.update(); + if (timer.isExpired(TIMER_DELTA_MS)) { + // resume from breakpoint if timeout happened when polling files + inFlightFilesCommitContextSet.add(commitContext); + final String message = + String.format( + "Timeout occurred when SubscriptionConsumer %s polling file %s with commit context %s, record writing offset %s for subsequent poll", + this, file.getAbsolutePath(), commitContext, writingOffset); + LOGGER.info(message); + throw new SubscriptionRuntimeNonCriticalException(message); + } + + final List responses = + pollFileInternal(commitContext, writingOffset, timer.remainingMs()); + + // If responses is empty, it means that some outdated subscription events may be being polled, + // so just return. + if (responses.isEmpty()) { + return Optional.empty(); + } + + // only one SubscriptionEvent polled currently + final SubscriptionPollResponse response = responses.get(0); + final SubscriptionPollPayload payload = response.getPayload(); + final short responseType = response.getResponseType(); + if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) { + final String errorMessage = String.format("unexpected response type: %s", responseType); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + switch (SubscriptionPollResponseType.valueOf(responseType)) { + case FILE_PIECE: + { + // check commit context + final SubscriptionCommitContext incomingCommitContext = response.getCommitContext(); + if (Objects.isNull(incomingCommitContext) + || !Objects.equals(commitContext, incomingCommitContext)) { + final String errorMessage = + String.format( + "inconsistent commit context, current is %s, incoming is %s, consumer: %s", + commitContext, incomingCommitContext, this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // check file name + if (!Objects.equals(rawFileName, ((FilePiecePayload) payload).getFileName())) { + final String errorMessage = + String.format( + "inconsistent file name, current is %s, incoming is %s, consumer: %s", + rawFileName, ((FilePiecePayload) payload).getFileName(), this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // write file piece + fileWriter.write(((FilePiecePayload) payload).getFilePiece()); + if (fileSaveFsync) { + fileWriter.getFD().sync(); + } + + // check offset + if (!Objects.equals( + fileWriter.length(), ((FilePiecePayload) payload).getNextWritingOffset())) { + final String errorMessage = + String.format( + "inconsistent file offset, current is %s, incoming is %s, consumer: %s", + fileWriter.length(), + ((FilePiecePayload) payload).getNextWritingOffset(), + this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // update offset + writingOffset = ((FilePiecePayload) payload).getNextWritingOffset(); + break; + } + case FILE_SEAL: + { + // check commit context + final SubscriptionCommitContext incomingCommitContext = response.getCommitContext(); + if (Objects.isNull(incomingCommitContext) + || !Objects.equals(commitContext, incomingCommitContext)) { + final String errorMessage = + String.format( + "inconsistent commit context, current is %s, incoming is %s, consumer: %s", + commitContext, incomingCommitContext, this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // check file name + if (!Objects.equals(rawFileName, ((FileSealPayload) payload).getFileName())) { + final String errorMessage = + String.format( + "inconsistent file name, current is %s, incoming is %s, consumer: %s", + rawFileName, ((FileSealPayload) payload).getFileName(), this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // check file length + if (fileWriter.length() != ((FileSealPayload) payload).getFileLength()) { + final String errorMessage = + String.format( + "inconsistent file length, current is %s, incoming is %s, consumer: %s", + fileWriter.length(), ((FileSealPayload) payload).getFileLength(), this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // optional sync and close + if (fileSaveFsync) { + fileWriter.getFD().sync(); + } + fileWriter.close(); + + LOGGER.info( + "SubscriptionConsumer {} successfully poll file {} with commit context {}", + this, + file.getAbsolutePath(), + commitContext); + + // generate subscription message + inFlightFilesCommitContextSet.remove(commitContext); + return Optional.of( + new SubscriptionMessage( + commitContext, + file.getAbsolutePath(), + ((FileSealPayload) payload).getDatabaseName())); + } + case ERROR: + { + // no need to check commit context + + final String errorMessage = ((ErrorPayload) payload).getErrorMessage(); + final boolean critical = ((ErrorPayload) payload).isCritical(); + if (!critical + && Objects.nonNull(errorMessage) + && errorMessage.contains(SubscriptionTimeoutException.KEYWORD)) { + // resume from breakpoint if timeout happened when polling files + inFlightFilesCommitContextSet.add(commitContext); + final String message = + String.format( + "Timeout occurred when SubscriptionConsumer %s polling file %s with commit context %s, record writing offset %s for subsequent poll", + this, file.getAbsolutePath(), commitContext, writingOffset); + LOGGER.info(message); + throw new SubscriptionPollTimeoutException(message); + } else { + LOGGER.warn( + "Error occurred when SubscriptionConsumer {} polling file {} with commit context {}: {}, critical: {}", + this, + file.getAbsolutePath(), + commitContext, + errorMessage, + critical); + if (critical) { + throw new SubscriptionRuntimeCriticalException(errorMessage); + } else { + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + } + } + default: + final String errorMessage = String.format("unexpected response type: %s", responseType); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + } + } + + private Optional pollTablets( + final SubscriptionPollResponse response, final PollTimer timer) throws SubscriptionException { + try { + return pollTabletsInternal(response, timer); + } catch (final Exception e) { + // construct temporary message to nack + nack( + Collections.singletonList( + new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()))); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + } + + private Optional pollTabletsInternal( + final SubscriptionPollResponse initialResponse, final PollTimer timer) { + final Map> tablets = + ((TabletsPayload) initialResponse.getPayload()).getTabletsWithDBInfo(); + final SubscriptionCommitContext commitContext = initialResponse.getCommitContext(); + + int nextOffset = ((TabletsPayload) initialResponse.getPayload()).getNextOffset(); + while (true) { + if (nextOffset <= 0) { + final int tabletsSize = tablets.values().stream().mapToInt(List::size).sum(); + if (!Objects.equals(tabletsSize, -nextOffset)) { + final String errorMessage = + String.format( + "inconsistent tablet size, current is %s, incoming is %s, consumer: %s", + tabletsSize, -nextOffset, this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + return Optional.of(new SubscriptionMessage(commitContext, tablets)); + } + + timer.update(); + if (timer.isExpired(TIMER_DELTA_MS)) { + final String errorMessage = + String.format( + "timeout while poll tablets with commit context: %s, consumer: %s", + commitContext, this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + final List responses = + pollTabletsInternal(commitContext, nextOffset, timer.remainingMs()); + + // If responses is empty, it means that some outdated subscription events may be being polled, + // so just return. + if (responses.isEmpty()) { + return Optional.empty(); + } + + // only one SubscriptionEvent polled currently + final SubscriptionPollResponse response = responses.get(0); + final SubscriptionPollPayload payload = response.getPayload(); + final short responseType = response.getResponseType(); + if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) { + final String errorMessage = String.format("unexpected response type: %s", responseType); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + switch (SubscriptionPollResponseType.valueOf(responseType)) { + case TABLETS: + { + // check commit context + final SubscriptionCommitContext incomingCommitContext = response.getCommitContext(); + if (Objects.isNull(incomingCommitContext) + || !Objects.equals(commitContext, incomingCommitContext)) { + final String errorMessage = + String.format( + "inconsistent commit context, current is %s, incoming is %s, consumer: %s", + commitContext, incomingCommitContext, this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + + // update tablets + for (Map.Entry> entry : + ((TabletsPayload) response.getPayload()).getTabletsWithDBInfo().entrySet()) { + tablets + .computeIfAbsent(entry.getKey(), databaseName -> new ArrayList<>()) + .addAll(entry.getValue()); + } + + // update offset + nextOffset = ((TabletsPayload) payload).getNextOffset(); + break; + } + case ERROR: + { + // no need to check commit context + + final String errorMessage = ((ErrorPayload) payload).getErrorMessage(); + final boolean critical = ((ErrorPayload) payload).isCritical(); + if (Objects.equals(payload, ErrorPayload.OUTDATED_ERROR_PAYLOAD)) { + // suppress warn log when poll outdated subscription event + return Optional.empty(); + } + LOGGER.warn( + "Error occurred when SubscriptionConsumer {} polling tablets with commit context {}: {}, critical: {}", + this, + commitContext, + errorMessage, + critical); + if (critical) { + throw new SubscriptionRuntimeCriticalException(errorMessage); + } else { + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + } + default: + final String errorMessage = String.format("unexpected response type: %s", responseType); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + } + } + + private List pollInternal( + final Set topicNames, final long timeoutMs) throws SubscriptionException { + providers.acquireReadLock(); + try { + final AbstractSubscriptionProvider provider = providers.getNextAvailableProvider(); + if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return Collections.emptyList(); + } + throw new SubscriptionConnectionException( + String.format( + "Cluster has no available subscription providers when %s poll topic %s", + this, topicNames)); + } + // ignore SubscriptionConnectionException to improve poll auto retry + try { + return provider.poll(topicNames, timeoutMs); + } catch (final SubscriptionConnectionException ignored) { + return Collections.emptyList(); + } + } finally { + providers.releaseReadLock(); + } + } + + private List pollFileInternal( + final SubscriptionCommitContext commitContext, final long writingOffset, final long timeoutMs) + throws SubscriptionException { + final int dataNodeId = commitContext.getDataNodeId(); + providers.acquireReadLock(); + try { + final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId); + if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return Collections.emptyList(); + } + throw new SubscriptionConnectionException( + String.format( + "something unexpected happened when %s poll file from subscription provider with data node id %s, the subscription provider may be unavailable or not existed", + this, dataNodeId)); + } + // ignore SubscriptionConnectionException to improve poll auto retry + try { + return provider.pollFile(commitContext, writingOffset, timeoutMs); + } catch (final SubscriptionConnectionException ignored) { + return Collections.emptyList(); + } + } finally { + providers.releaseReadLock(); + } + } + + private List pollTabletsInternal( + final SubscriptionCommitContext commitContext, final int offset, final long timeoutMs) + throws SubscriptionException { + final int dataNodeId = commitContext.getDataNodeId(); + providers.acquireReadLock(); + try { + final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId); + if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return Collections.emptyList(); + } + throw new SubscriptionConnectionException( + String.format( + "something unexpected happened when %s poll tablets from subscription provider with data node id %s, the subscription provider may be unavailable or not existed", + this, dataNodeId)); + } + // ignore SubscriptionConnectionException to improve poll auto retry + try { + return provider.pollTablets(commitContext, offset, timeoutMs); + } catch (final SubscriptionConnectionException ignored) { + return Collections.emptyList(); + } + } finally { + providers.releaseReadLock(); + } + } + + /////////////////////////////// commit sync (ack & nack) /////////////////////////////// + + protected void ack(final Iterable messages) throws SubscriptionException { + final Map> dataNodeIdToSubscriptionCommitContexts = + new HashMap<>(); + for (final SubscriptionMessage message : messages) { + dataNodeIdToSubscriptionCommitContexts + .computeIfAbsent(message.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) + .add(message.getCommitContext()); + } + for (final Entry> entry : + dataNodeIdToSubscriptionCommitContexts.entrySet()) { + commitInternal(entry.getKey(), entry.getValue(), false); + } + } + + protected void nack(final Iterable messages) throws SubscriptionException { + final Map> dataNodeIdToSubscriptionCommitContexts = + new HashMap<>(); + for (final SubscriptionMessage message : messages) { + // make every effort to delete stale intermediate file + if (Objects.equals( + SubscriptionMessageType.TS_FILE_HANDLER.getType(), message.getMessageType()) + && + // do not delete file that can resume from breakpoint + !inFlightFilesCommitContextSet.contains(message.getCommitContext())) { + try { + message.getTsFileHandler().deleteFile(); + } catch (final Exception ignored) { + } + } + dataNodeIdToSubscriptionCommitContexts + .computeIfAbsent(message.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) + .add(message.getCommitContext()); + } + for (final Entry> entry : + dataNodeIdToSubscriptionCommitContexts.entrySet()) { + commitInternal(entry.getKey(), entry.getValue(), true); + } + } + + private void nack(final List responses) throws SubscriptionException { + final Map> dataNodeIdToSubscriptionCommitContexts = + new HashMap<>(); + for (final SubscriptionPollResponse response : responses) { + // there is no stale intermediate file here + dataNodeIdToSubscriptionCommitContexts + .computeIfAbsent(response.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) + .add(response.getCommitContext()); + } + for (final Entry> entry : + dataNodeIdToSubscriptionCommitContexts.entrySet()) { + commitInternal(entry.getKey(), entry.getValue(), true); + } + } + + private void commitInternal( + final int dataNodeId, + final List subscriptionCommitContexts, + final boolean nack) + throws SubscriptionException { + providers.acquireReadLock(); + try { + final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId); + if (Objects.isNull(provider) || !provider.isAvailable()) { + if (isClosed()) { + return; + } + throw new SubscriptionConnectionException( + String.format( + "something unexpected happened when %s commit (nack: %s) messages to subscription provider with data node id %s, the subscription provider may be unavailable or not existed", + this, nack, dataNodeId)); + } + provider.commit(subscriptionCommitContexts, nack); + } finally { + providers.releaseReadLock(); + } + } + + /////////////////////////////// heartbeat /////////////////////////////// + + private void submitHeartbeatWorker() { + final ScheduledFuture[] future = new ScheduledFuture[1]; + future[0] = + SubscriptionExecutorServiceManager.submitHeartbeatWorker( + () -> { + if (isClosed()) { + if (Objects.nonNull(future[0])) { + future[0].cancel(false); + LOGGER.info("SubscriptionConsumer {} cancel heartbeat worker", this); + } + return; + } + providers.heartbeat(this); + }, + heartbeatIntervalMs); + LOGGER.info("SubscriptionConsumer {} submit heartbeat worker", this); + } + + /////////////////////////////// sync endpoints /////////////////////////////// + + private void submitEndpointsSyncer() { + final ScheduledFuture[] future = new ScheduledFuture[1]; + future[0] = + SubscriptionExecutorServiceManager.submitEndpointsSyncer( + () -> { + if (isClosed()) { + if (Objects.nonNull(future[0])) { + future[0].cancel(false); + LOGGER.info("SubscriptionConsumer {} cancel endpoints syncer", this); + } + return; + } + providers.sync(this); + }, + endpointsSyncIntervalMs); + LOGGER.info("SubscriptionConsumer {} submit endpoints syncer", this); + } + + /////////////////////////////// commit async /////////////////////////////// + + protected void commitAsync( + final Iterable messages, final AsyncCommitCallback callback) { + SubscriptionExecutorServiceManager.submitAsyncCommitWorker( + new AsyncCommitWorker(messages, callback)); + } + + private class AsyncCommitWorker implements Runnable { + + private final Iterable messages; + private final AsyncCommitCallback callback; + + public AsyncCommitWorker( + final Iterable messages, final AsyncCommitCallback callback) { + this.messages = messages; + this.callback = callback; + } + + @Override + public void run() { + if (isClosed()) { + return; + } + + try { + ack(messages); + callback.onComplete(); + } catch (final Exception e) { + callback.onFailure(e); + } + } + } + + protected CompletableFuture commitAsync(final Iterable messages) { + final CompletableFuture future = new CompletableFuture<>(); + SubscriptionExecutorServiceManager.submitAsyncCommitWorker( + () -> { + if (isClosed()) { + return; + } + + try { + ack(messages); + future.complete(null); + } catch (final Throwable e) { + future.completeExceptionally(e); + } + }); + return future; + } + + /////////////////////////////// redirection /////////////////////////////// + + private void subscribeWithRedirection(final Set topicNames) throws SubscriptionException { + final List providers = this.providers.getAllAvailableProviders(); + if (providers.isEmpty()) { + throw new SubscriptionConnectionException( + String.format( + "Cluster has no available subscription providers when %s subscribe topic %s", + this, topicNames)); + } + for (final AbstractSubscriptionProvider provider : providers) { + try { + subscribedTopics = provider.subscribe(topicNames); + return; + } catch (final Exception e) { + if (e instanceof SubscriptionPipeTimeoutException) { + // degrade exception to log for pipe timeout + LOGGER.warn(e.getMessage()); + return; + } + LOGGER.warn( + "{} failed to subscribe topics {} from subscription provider {}, try next subscription provider...", + this, + topicNames, + provider, + e); + } + } + final String errorMessage = + String.format( + "%s failed to subscribe topics %s from all available subscription providers %s", + this, topicNames, providers); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeCriticalException(errorMessage); + } + + private void unsubscribeWithRedirection(final Set topicNames) + throws SubscriptionException { + final List providers = this.providers.getAllAvailableProviders(); + if (providers.isEmpty()) { + throw new SubscriptionConnectionException( + String.format( + "Cluster has no available subscription providers when %s unsubscribe topic %s", + this, topicNames)); + } + for (final AbstractSubscriptionProvider provider : providers) { + try { + subscribedTopics = provider.unsubscribe(topicNames); + return; + } catch (final Exception e) { + if (e instanceof SubscriptionPipeTimeoutException) { + // degrade exception to log for pipe timeout + LOGGER.warn(e.getMessage()); + return; + } + LOGGER.warn( + "{} failed to unsubscribe topics {} from subscription provider {}, try next subscription provider...", + this, + topicNames, + provider, + e); + } + } + final String errorMessage = + String.format( + "%s failed to unsubscribe topics %s from all available subscription providers %s", + this, topicNames, providers); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeCriticalException(errorMessage); + } + + Map fetchAllEndPointsWithRedirection() throws SubscriptionException { + final List providers = this.providers.getAllAvailableProviders(); + if (providers.isEmpty()) { + throw new SubscriptionConnectionException( + String.format( + "Cluster has no available subscription providers when %s fetch all endpoints", this)); + } + for (final AbstractSubscriptionProvider provider : providers) { + try { + return provider.heartbeat().getEndPoints(); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to fetch all endpoints from subscription provider {}, try next subscription provider...", + this, + provider, + e); + } + } + final String errorMessage = + String.format( + "%s failed to fetch all endpoints from all available subscription providers %s", + this, providers); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeCriticalException(errorMessage); + } + + /////////////////////////////// stringify /////////////////////////////// + + protected Map coreReportMessage() { + final Map result = new HashMap<>(); + result.put("consumerId", consumerId); + result.put("consumerGroupId", consumerGroupId); + result.put("isClosed", isClosed.toString()); + result.put("fileSaveDir", fileSaveDir); + result.put( + "inFlightFilesCommitContextSet", + CollectionUtils.getLimitedString(inFlightFilesCommitContextSet, 32)); + result.put( + "subscribedTopicNames", CollectionUtils.getLimitedString(subscribedTopics.keySet(), 32)); + return result; + } + + protected Map allReportMessage() { + final Map result = new HashMap<>(); + result.put("consumerId", consumerId); + result.put("consumerGroupId", consumerGroupId); + result.put("heartbeatIntervalMs", String.valueOf(heartbeatIntervalMs)); + result.put("endpointsSyncIntervalMs", String.valueOf(endpointsSyncIntervalMs)); + result.put("providers", providers.toString()); + result.put("isClosed", isClosed.toString()); + result.put("isReleased", isReleased.toString()); + result.put("fileSaveDir", fileSaveDir); + result.put("fileSaveFsync", String.valueOf(fileSaveFsync)); + result.put("inFlightFilesCommitContextSet", inFlightFilesCommitContextSet.toString()); + result.put("thriftMaxFrameSize", String.valueOf(thriftMaxFrameSize)); + result.put("maxPollParallelism", String.valueOf(maxPollParallelism)); + result.put("subscribedTopics", subscribedTopics.toString()); + return result; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerBuilder.java new file mode 100644 index 0000000000000..7f965069e73e2 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerBuilder.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; + +import org.apache.thrift.annotation.Nullable; + +import java.util.List; +import java.util.Objects; + +public class AbstractSubscriptionConsumerBuilder { + + protected String host; + protected Integer port; + protected List nodeUrls; + + protected String username = SessionConfig.DEFAULT_USER; + protected String password = SessionConfig.DEFAULT_PASSWORD; + + protected String consumerId; + protected String consumerGroupId; + + protected long heartbeatIntervalMs = ConsumerConstant.HEARTBEAT_INTERVAL_MS_DEFAULT_VALUE; + protected long endpointsSyncIntervalMs = + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_DEFAULT_VALUE; + + protected String fileSaveDir = ConsumerConstant.FILE_SAVE_DIR_DEFAULT_VALUE; + protected boolean fileSaveFsync = ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE; + + protected int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; + protected int maxPollParallelism = ConsumerConstant.MAX_POLL_PARALLELISM_DEFAULT_VALUE; + + public AbstractSubscriptionConsumerBuilder host(final String host) { + this.host = host; + return this; + } + + public AbstractSubscriptionConsumerBuilder port(final Integer port) { + this.port = port; + return this; + } + + public AbstractSubscriptionConsumerBuilder nodeUrls(final List nodeUrls) { + this.nodeUrls = nodeUrls; + return this; + } + + public AbstractSubscriptionConsumerBuilder username(final String username) { + this.username = username; + return this; + } + + public AbstractSubscriptionConsumerBuilder password(final String password) { + this.password = password; + return this; + } + + public AbstractSubscriptionConsumerBuilder consumerId(@Nullable final String consumerId) { + if (Objects.isNull(consumerId)) { + return this; + } + this.consumerId = IdentifierUtils.checkAndParseIdentifier(consumerId); + return this; + } + + public AbstractSubscriptionConsumerBuilder consumerGroupId( + @Nullable final String consumerGroupId) { + if (Objects.isNull(consumerGroupId)) { + return this; + } + this.consumerGroupId = IdentifierUtils.checkAndParseIdentifier(consumerGroupId); + return this; + } + + public AbstractSubscriptionConsumerBuilder heartbeatIntervalMs(final long heartbeatIntervalMs) { + this.heartbeatIntervalMs = + Math.max(heartbeatIntervalMs, ConsumerConstant.HEARTBEAT_INTERVAL_MS_MIN_VALUE); + return this; + } + + public AbstractSubscriptionConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + this.endpointsSyncIntervalMs = + Math.max(endpointsSyncIntervalMs, ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE); + return this; + } + + public AbstractSubscriptionConsumerBuilder fileSaveDir(final String fileSaveDir) { + this.fileSaveDir = fileSaveDir; + return this; + } + + public AbstractSubscriptionConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + this.fileSaveFsync = fileSaveFsync; + return this; + } + + public AbstractSubscriptionConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + this.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + public AbstractSubscriptionConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + // Here the minimum value of max poll parallelism is set to 1 instead of 0, in order to use a + // single thread to execute poll whenever there are idle resources available, thereby + // achieving strict timeout. + this.maxPollParallelism = Math.max(maxPollParallelism, 1); + return this; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java new file mode 100644 index 0000000000000..df5dfcfd7f740 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionPipeTimeoutException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; +import org.apache.iotdb.rpc.subscription.payload.poll.PollFilePayload; +import org.apache.iotdb.rpc.subscription.payload.poll.PollPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.PollTabletsPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequest; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequestType; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeCloseReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeCommitReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHandshakeReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHeartbeatReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribePollReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeSubscribeReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeUnsubscribeReq; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHandshakeResp; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribePollResp; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeSubscribeResp; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeUnsubscribeResp; +import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.iotdb.session.AbstractSessionBuilder; +import org.apache.iotdb.session.subscription.SubscriptionSessionConnection; +import org.apache.iotdb.session.subscription.SubscriptionSessionWrapper; + +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractSubscriptionProvider { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSubscriptionProvider.class); + + private static final String STATUS_FORMATTER = "Status code is [%s], status message is [%s]."; + private static final String INTERNAL_ERROR_FORMATTER = + "Internal error occurred. " + STATUS_FORMATTER; + private static final String SUBSCRIPTION_PIPE_TIMEOUT_FORMATTER = + "A timeout has occurred in procedures related to the pipe within the subscription procedure. " + + "Please manually check the subscription correctness later. " + + STATUS_FORMATTER; + + private final SubscriptionSessionWrapper session; + + private String consumerId; + private String consumerGroupId; + + private final AtomicBoolean isClosed = new AtomicBoolean(true); + private final AtomicBoolean isAvailable = new AtomicBoolean(false); + + private final TEndPoint endPoint; + private int dataNodeId; + + private final String username; + private final String password; + + protected abstract AbstractSessionBuilder constructSubscriptionSessionBuilder( + final String host, + final int port, + final String username, + final String password, + final int thriftMaxFrameSize); + + protected AbstractSubscriptionProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + this.session = + new SubscriptionSessionWrapper( + constructSubscriptionSessionBuilder( + endPoint.ip, endPoint.port, username, password, thriftMaxFrameSize)); + this.endPoint = endPoint; + this.consumerId = consumerId; + this.consumerGroupId = consumerGroupId; + this.username = username; + this.password = password; + } + + SubscriptionSessionConnection getSessionConnection() { + return session.getSessionConnection(); + } + + boolean isAvailable() { + return isAvailable.get(); + } + + void setAvailable() { + isAvailable.set(true); + } + + void setUnavailable() { + isAvailable.set(false); + } + + int getDataNodeId() { + return dataNodeId; + } + + String getConsumerId() { + return consumerId; + } + + String getConsumerGroupId() { + return consumerGroupId; + } + + TEndPoint getEndPoint() { + return endPoint; + } + + /////////////////////////////// open & close /////////////////////////////// + + synchronized void handshake() throws SubscriptionException, IoTDBConnectionException { + if (!isClosed.get()) { + return; + } + + session.open(); // throw IoTDBConnectionException + + // TODO: pass the complete consumer parameter configuration to the server + final Map consumerAttributes = new HashMap<>(); + consumerAttributes.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, consumerGroupId); + consumerAttributes.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); + consumerAttributes.put(ConsumerConstant.USERNAME_KEY, username); + consumerAttributes.put(ConsumerConstant.PASSWORD_KEY, password); + consumerAttributes.put(ConsumerConstant.SQL_DIALECT_KEY, session.getSqlDialect()); + + final PipeSubscribeHandshakeResp resp = + handshake(new ConsumerConfig(consumerAttributes)); // throw SubscriptionException + dataNodeId = resp.getDataNodeId(); + consumerId = resp.getConsumerId(); + consumerGroupId = resp.getConsumerGroupId(); + + isClosed.set(false); + setAvailable(); + } + + PipeSubscribeHandshakeResp handshake(final ConsumerConfig consumerConfig) + throws SubscriptionException { + final PipeSubscribeHandshakeReq req; + try { + req = PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig); + } catch (final IOException e) { + LOGGER.warn( + "IOException occurred when SubscriptionProvider {} serialize handshake request {}", + this, + consumerConfig, + e); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(req); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} handshake with request {}, set SubscriptionProvider unavailable", + this, + consumerConfig, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + return PipeSubscribeHandshakeResp.fromTPipeSubscribeResp(resp); + } + + synchronized void close() throws SubscriptionException, IoTDBConnectionException { + if (isClosed.get()) { + return; + } + + try { + closeInternal(); // throw SubscriptionException + } finally { + session.close(); // throw IoTDBConnectionException + setUnavailable(); + isClosed.set(true); + } + } + + void closeInternal() throws SubscriptionException { + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(PipeSubscribeCloseReq.toTPipeSubscribeReq()); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} close, set SubscriptionProvider unavailable", + this, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + } + + /////////////////////////////// subscription APIs /////////////////////////////// + + PipeSubscribeHeartbeatResp heartbeat() throws SubscriptionException { + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(PipeSubscribeHeartbeatReq.toTPipeSubscribeReq()); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} heartbeat, set SubscriptionProvider unavailable", + this, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + return PipeSubscribeHeartbeatResp.fromTPipeSubscribeResp(resp); + } + + Map subscribe(final Set topicNames) throws SubscriptionException { + final PipeSubscribeSubscribeReq req; + try { + req = PipeSubscribeSubscribeReq.toTPipeSubscribeReq(topicNames); + } catch (final IOException e) { + LOGGER.warn( + "IOException occurred when SubscriptionProvider {} serialize subscribe request {}", + this, + topicNames, + e); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(req); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} subscribe with request {}, set SubscriptionProvider unavailable", + this, + topicNames, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + final PipeSubscribeSubscribeResp subscribeResp = + PipeSubscribeSubscribeResp.fromTPipeSubscribeResp(resp); + return subscribeResp.getTopics(); + } + + Map unsubscribe(final Set topicNames) throws SubscriptionException { + final PipeSubscribeUnsubscribeReq req; + try { + req = PipeSubscribeUnsubscribeReq.toTPipeSubscribeReq(topicNames); + } catch (final IOException e) { + LOGGER.warn( + "IOException occurred when SubscriptionProvider {} serialize unsubscribe request {}", + this, + topicNames, + e); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(req); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} unsubscribe with request {}, set SubscriptionProvider unavailable", + this, + topicNames, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + final PipeSubscribeUnsubscribeResp unsubscribeResp = + PipeSubscribeUnsubscribeResp.fromTPipeSubscribeResp(resp); + return unsubscribeResp.getTopics(); + } + + List poll(final Set topicNames, final long timeoutMs) + throws SubscriptionException { + return poll( + new SubscriptionPollRequest( + SubscriptionPollRequestType.POLL.getType(), + new PollPayload(topicNames), + timeoutMs, + session.getThriftMaxFrameSize())); + } + + List pollFile( + final SubscriptionCommitContext commitContext, final long writingOffset, final long timeoutMs) + throws SubscriptionException { + return poll( + new SubscriptionPollRequest( + SubscriptionPollRequestType.POLL_FILE.getType(), + new PollFilePayload(commitContext, writingOffset), + timeoutMs, + session.getThriftMaxFrameSize())); + } + + List pollTablets( + final SubscriptionCommitContext commitContext, final int offset, final long timeoutMs) + throws SubscriptionException { + return poll( + new SubscriptionPollRequest( + SubscriptionPollRequestType.POLL_TABLETS.getType(), + new PollTabletsPayload(commitContext, offset), + timeoutMs, + session.getThriftMaxFrameSize())); + } + + List poll(final SubscriptionPollRequest pollMessage) + throws SubscriptionException { + final PipeSubscribePollReq req; + try { + req = PipeSubscribePollReq.toTPipeSubscribeReq(pollMessage); + } catch (final IOException e) { + LOGGER.warn( + "IOException occurred when SubscriptionProvider {} serialize poll request {}", + this, + pollMessage, + e); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(req); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} poll with request {}, set SubscriptionProvider unavailable", + this, + pollMessage, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + final PipeSubscribePollResp pollResp = PipeSubscribePollResp.fromTPipeSubscribeResp(resp); + return pollResp.getResponses(); + } + + void commit(final List subscriptionCommitContexts, final boolean nack) + throws SubscriptionException { + final PipeSubscribeCommitReq req; + try { + req = PipeSubscribeCommitReq.toTPipeSubscribeReq(subscriptionCommitContexts, nack); + } catch (final IOException e) { + LOGGER.warn( + "IOException occurred when SubscriptionProvider {} serialize commit request {}", + this, + subscriptionCommitContexts, + e); + throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); + } + final TPipeSubscribeResp resp; + try { + resp = getSessionConnection().pipeSubscribe(req); + } catch (final TException e) { + // Assume provider unavailable + LOGGER.warn( + "TException occurred when SubscriptionProvider {} commit with request {}, set SubscriptionProvider unavailable", + this, + subscriptionCommitContexts, + e); + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + verifyPipeSubscribeSuccess(resp.status); + } + + private static void verifyPipeSubscribeSuccess(final TSStatus status) + throws SubscriptionException { + switch (status.code) { + case 200: // SUCCESS_STATUS + return; + case 1902: // SUBSCRIPTION_HANDSHAKE_ERROR + case 1903: // SUBSCRIPTION_HEARTBEAT_ERROR + case 1904: // SUBSCRIPTION_POLL_ERROR + case 1905: // SUBSCRIPTION_COMMIT_ERROR + case 1906: // SUBSCRIPTION_CLOSE_ERROR + case 1907: // SUBSCRIPTION_SUBSCRIBE_ERROR + case 1908: // SUBSCRIPTION_UNSUBSCRIBE_ERROR + { + final String errorMessage = + String.format(INTERNAL_ERROR_FORMATTER, status.code, status.message); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + case 1911: // SUBSCRIPTION_PIPE_TIMEOUT_ERROR + throw new SubscriptionPipeTimeoutException( + String.format(SUBSCRIPTION_PIPE_TIMEOUT_FORMATTER, status.code, status.message)); + case 1900: // SUBSCRIPTION_VERSION_ERROR + case 1901: // SUBSCRIPTION_TYPE_ERROR + case 1909: // SUBSCRIPTION_MISSING_CUSTOMER + default: + { + final String errorMessage = + String.format(INTERNAL_ERROR_FORMATTER, status.code, status.message); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeCriticalException(status.message); + } + } + } + + @Override + public String toString() { + return "SubscriptionProvider{endPoint=" + + endPoint + + ", dataNodeId=" + + dataNodeId + + ", consumerId=" + + consumerId + + ", consumerGroupId=" + + consumerGroupId + + ", isAvailable=" + + isAvailable + + ", isClosed=" + + isClosed + + "}"; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java new file mode 100644 index 0000000000000..fe765dab2dae5 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +final class AbstractSubscriptionProviders { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSubscriptionProviders.class); + + private final SortedMap subscriptionProviders = + new ConcurrentSkipListMap<>(); + private int nextDataNodeId = -1; + + private final ReentrantReadWriteLock subscriptionProvidersLock = new ReentrantReadWriteLock(true); + + private final Set initialEndpoints; + + AbstractSubscriptionProviders(final Set initialEndpoints) { + this.initialEndpoints = initialEndpoints; + } + + /////////////////////////////// lock /////////////////////////////// + + void acquireReadLock() { + subscriptionProvidersLock.readLock().lock(); + } + + void releaseReadLock() { + subscriptionProvidersLock.readLock().unlock(); + } + + void acquireWriteLock() { + subscriptionProvidersLock.writeLock().lock(); + } + + void releaseWriteLock() { + subscriptionProvidersLock.writeLock().unlock(); + } + + /////////////////////////////// CRUD /////////////////////////////// + + /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ + void openProviders(final AbstractSubscriptionConsumer consumer) throws SubscriptionException { + // close stale providers + closeProviders(); + + for (final TEndPoint endPoint : initialEndpoints) { + final AbstractSubscriptionProvider defaultProvider; + final int defaultDataNodeId; + + try { + defaultProvider = consumer.constructProviderAndHandshake(endPoint); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to create connection with {} because of {}", consumer, endPoint, e, e); + continue; // try next endpoint + } + defaultDataNodeId = defaultProvider.getDataNodeId(); + addProvider(defaultDataNodeId, defaultProvider); + + final Map allEndPoints; + try { + allEndPoints = defaultProvider.heartbeat().getEndPoints(); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to fetch all endpoints from {} because of {}", consumer, endPoint, e, e); + break; + } + + for (final Map.Entry entry : allEndPoints.entrySet()) { + if (defaultDataNodeId == entry.getKey()) { + continue; + } + + final AbstractSubscriptionProvider provider; + try { + provider = consumer.constructProviderAndHandshake(entry.getValue()); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to create connection with {} because of {}", + consumer, + entry.getValue(), + e, + e); + continue; + } + addProvider(entry.getKey(), provider); + } + + break; + } + + if (hasNoAvailableProviders()) { + throw new SubscriptionConnectionException( + String.format( + "Cluster has no available subscription providers to connect with initial endpoints %s", + initialEndpoints)); + } + + nextDataNodeId = subscriptionProviders.firstKey(); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ + void closeProviders() { + for (final AbstractSubscriptionProvider provider : getAllProviders()) { + try { + provider.close(); + } catch (final Exception e) { + LOGGER.warn("Failed to close subscription provider {} because of {}", provider, e, e); + } + } + subscriptionProviders.clear(); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ + void addProvider(final int dataNodeId, final AbstractSubscriptionProvider provider) { + // the subscription provider is opened + LOGGER.info("add new subscription provider {}", provider); + subscriptionProviders.put(dataNodeId, provider); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ + void closeAndRemoveProvider(final int dataNodeId) + throws SubscriptionException, IoTDBConnectionException { + if (!containsProvider(dataNodeId)) { + return; + } + final AbstractSubscriptionProvider provider = subscriptionProviders.get(dataNodeId); + try { + provider.close(); + } finally { + LOGGER.info("close and remove stale subscription provider {}", provider); + subscriptionProviders.remove(dataNodeId); + } + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + boolean hasNoProviders() { + return subscriptionProviders.isEmpty(); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + List getAllProviders() { + return new ArrayList<>(subscriptionProviders.values()); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + AbstractSubscriptionProvider getProvider(final int dataNodeId) { + return containsProvider(dataNodeId) ? subscriptionProviders.get(dataNodeId) : null; + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + boolean hasNoAvailableProviders() { + return subscriptionProviders.values().stream() + .noneMatch(AbstractSubscriptionProvider::isAvailable); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + boolean containsProvider(final int dataNodeId) { + return subscriptionProviders.containsKey(dataNodeId); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + List getAllAvailableProviders() { + return subscriptionProviders.values().stream() + .filter(AbstractSubscriptionProvider::isAvailable) + .collect(Collectors.toList()); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + void updateNextDataNodeId() { + final SortedMap subProviders = + subscriptionProviders.tailMap(nextDataNodeId + 1); + nextDataNodeId = + subProviders.isEmpty() ? subscriptionProviders.firstKey() : subProviders.firstKey(); + } + + /** Caller should ensure that the method is called in the lock {@link #acquireReadLock()}. */ + AbstractSubscriptionProvider getNextAvailableProvider() { + if (hasNoAvailableProviders()) { + return null; + } + + AbstractSubscriptionProvider provider; + provider = getProvider(nextDataNodeId); + while (Objects.isNull(provider) || !provider.isAvailable()) { + updateNextDataNodeId(); + provider = getProvider(nextDataNodeId); + } + updateNextDataNodeId(); + + return provider; + } + + /////////////////////////////// heartbeat /////////////////////////////// + + void heartbeat(final AbstractSubscriptionConsumer consumer) { + if (consumer.isClosed()) { + return; + } + + acquireWriteLock(); + try { + heartbeatInternal(consumer); + } finally { + releaseWriteLock(); + } + } + + private void heartbeatInternal(final AbstractSubscriptionConsumer consumer) { + for (final AbstractSubscriptionProvider provider : getAllProviders()) { + try { + consumer.subscribedTopics = provider.heartbeat().getTopics(); + provider.setAvailable(); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to sending heartbeat to subscription provider {} because of {}, set subscription provider unavailable", + consumer, + provider, + e, + e); + provider.setUnavailable(); + } + } + } + + /////////////////////////////// sync endpoints /////////////////////////////// + + void sync(final AbstractSubscriptionConsumer consumer) { + if (consumer.isClosed()) { + return; + } + + acquireWriteLock(); + try { + syncInternal(consumer); + } finally { + releaseWriteLock(); + } + } + + private void syncInternal(final AbstractSubscriptionConsumer consumer) { + if (hasNoAvailableProviders()) { + try { + openProviders(consumer); + } catch (final Exception e) { + LOGGER.warn("Failed to open providers for consumer {} because of {}", consumer, e, e); + return; + } + } + + final Map allEndPoints; + try { + allEndPoints = consumer.fetchAllEndPointsWithRedirection(); + } catch (final Exception e) { + LOGGER.warn("Failed to fetch all endpoints for consumer {} because of {}", consumer, e, e); + return; + } + + // add new providers or handshake existing providers + for (final Map.Entry entry : allEndPoints.entrySet()) { + final AbstractSubscriptionProvider provider = getProvider(entry.getKey()); + if (Objects.isNull(provider)) { + // new provider + final TEndPoint endPoint = entry.getValue(); + final AbstractSubscriptionProvider newProvider; + try { + newProvider = consumer.constructProviderAndHandshake(endPoint); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to create connection with {} because of {}", consumer, endPoint, e, e); + continue; + } + addProvider(entry.getKey(), newProvider); + } else { + // existing provider + try { + consumer.subscribedTopics = provider.heartbeat().getTopics(); + provider.setAvailable(); + } catch (final Exception e) { + LOGGER.warn( + "{} failed to sending heartbeat to subscription provider {} because of {}, set subscription provider unavailable", + consumer, + provider, + e, + e); + provider.setUnavailable(); + } + // close and remove unavailable provider (reset the connection as much as possible) + if (!provider.isAvailable()) { + try { + closeAndRemoveProvider(entry.getKey()); + } catch (final Exception e) { + LOGGER.warn( + "Exception occurred when {} closing and removing subscription provider {} because of {}", + consumer, + provider, + e, + e); + } + } + } + } + + // close and remove stale providers + for (final AbstractSubscriptionProvider provider : getAllProviders()) { + final int dataNodeId = provider.getDataNodeId(); + if (!allEndPoints.containsKey(dataNodeId)) { + try { + closeAndRemoveProvider(dataNodeId); + } catch (final Exception e) { + LOGGER.warn( + "Exception occurred when {} closing and removing subscription provider {} because of {}", + consumer, + provider, + e, + e); + } + } + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("SubscriptionProviders{"); + for (final Map.Entry entry : + subscriptionProviders.entrySet()) { + sb.append(entry.getValue().toString()).append(", "); + } + if (!subscriptionProviders.isEmpty()) { + sb.delete(sb.length() - 2, sb.length()); + } + sb.append("}"); + return sb.toString(); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java new file mode 100644 index 0000000000000..0c7478fa64dfb --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.util.CollectionUtils; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +/** + * The {@link AbstractSubscriptionPullConsumer} corresponds to the pull consumption mode in the + * message queue. + * + *

User code needs to actively call the data retrieval logic, i.e., the {@link #poll} method. + * + *

Auto-commit for consumption progress can be configured in {@link #autoCommit}. + * + *

NOTE: It is not recommended to use the {@link #poll} method with the same consumer in a + * multithreaded environment. Instead, it is advised to increase the number of consumers to improve + * data retrieval parallelism. + */ +public abstract class AbstractSubscriptionPullConsumer extends AbstractSubscriptionConsumer { + + private static final Logger LOGGER = + LoggerFactory.getLogger(AbstractSubscriptionPullConsumer.class); + + private final boolean autoCommit; + private final long autoCommitIntervalMs; + + private SortedMap> uncommittedMessages; + + private final AtomicBoolean isClosed = new AtomicBoolean(true); + + @Override + boolean isClosed() { + return isClosed.get(); + } + + /////////////////////////////// ctor /////////////////////////////// + + protected AbstractSubscriptionPullConsumer( + final AbstractSubscriptionPullConsumerBuilder builder) { + super(builder); + + this.autoCommit = builder.autoCommit; + this.autoCommitIntervalMs = builder.autoCommitIntervalMs; + } + + public AbstractSubscriptionPullConsumer(final Properties properties) { + this( + properties, + (Boolean) + properties.getOrDefault( + ConsumerConstant.AUTO_COMMIT_KEY, ConsumerConstant.AUTO_COMMIT_DEFAULT_VALUE), + (Long) + properties.getOrDefault( + ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_KEY, + ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_DEFAULT_VALUE)); + } + + protected AbstractSubscriptionPullConsumer( + final Properties properties, final boolean autoCommit, final long autoCommitIntervalMs) { + super( + new AbstractSubscriptionPullConsumerBuilder() + .autoCommit(autoCommit) + .autoCommitIntervalMs(autoCommitIntervalMs), + properties); + + this.autoCommit = autoCommit; + this.autoCommitIntervalMs = + Math.max(autoCommitIntervalMs, ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_MIN_VALUE); + } + + /////////////////////////////// open & close /////////////////////////////// + + @Override + protected synchronized void open() throws SubscriptionException { + if (!isClosed.get()) { + return; + } + + super.open(); + + // set isClosed to false before submitting workers + isClosed.set(false); + + // submit auto poll worker if enabling auto commit + if (autoCommit) { + uncommittedMessages = new ConcurrentSkipListMap<>(); + submitAutoCommitWorker(); + } + } + + @Override + public synchronized void close() { + if (isClosed.get()) { + return; + } + + if (autoCommit) { + // commit all uncommitted messages + commitAllUncommittedMessages(); + } + + super.close(); + isClosed.set(true); + } + + /////////////////////////////// poll & commit /////////////////////////////// + + protected List poll(final Duration timeout) throws SubscriptionException { + return poll(Collections.emptySet(), timeout.toMillis()); + } + + protected List poll(final long timeoutMs) throws SubscriptionException { + return poll(Collections.emptySet(), timeoutMs); + } + + protected List poll(final Set topicNames, final Duration timeout) + throws SubscriptionException { + return poll(topicNames, timeout.toMillis()); + } + + protected List poll(final Set topicNames, final long timeoutMs) + throws SubscriptionException { + // parse topic names from external source + Set parsedTopicNames = + topicNames.stream() + .map(IdentifierUtils::checkAndParseIdentifier) + .collect(Collectors.toSet()); + + if (!parsedTopicNames.isEmpty()) { + // filter unsubscribed topics + parsedTopicNames.stream() + .filter(topicName -> !subscribedTopics.containsKey(topicName)) + .forEach( + topicName -> + LOGGER.warn( + "SubscriptionPullConsumer {} does not subscribe to topic {}", + this, + topicName)); + } else { + parsedTopicNames = subscribedTopics.keySet(); + } + + if (parsedTopicNames.isEmpty()) { + return Collections.emptyList(); + } + + final List messages = multiplePoll(parsedTopicNames, timeoutMs); + if (messages.isEmpty()) { + LOGGER.info( + "SubscriptionPullConsumer {} poll empty message from topics {} after {} millisecond(s)", + this, + CollectionUtils.getLimitedString(parsedTopicNames, 32), + timeoutMs); + return messages; + } + + // add to uncommitted messages + if (autoCommit) { + final long currentTimestamp = System.currentTimeMillis(); + long index = currentTimestamp / autoCommitIntervalMs; + if (currentTimestamp % autoCommitIntervalMs == 0) { + index -= 1; + } + uncommittedMessages + .computeIfAbsent(index, o -> new ConcurrentSkipListSet<>()) + .addAll(messages); + } + + return messages; + } + + /////////////////////////////// commit /////////////////////////////// + + protected void commitSync(final SubscriptionMessage message) throws SubscriptionException { + super.ack(Collections.singletonList(message)); + } + + protected void commitSync(final Iterable messages) + throws SubscriptionException { + super.ack(messages); + } + + protected CompletableFuture commitAsync(final SubscriptionMessage message) { + return super.commitAsync(Collections.singletonList(message)); + } + + protected CompletableFuture commitAsync(final Iterable messages) { + return super.commitAsync(messages); + } + + protected void commitAsync( + final SubscriptionMessage message, final AsyncCommitCallback callback) { + super.commitAsync(Collections.singletonList(message), callback); + } + + protected void commitAsync( + final Iterable messages, final AsyncCommitCallback callback) { + super.commitAsync(messages, callback); + } + + /////////////////////////////// auto commit /////////////////////////////// + + private void submitAutoCommitWorker() { + final ScheduledFuture[] future = new ScheduledFuture[1]; + future[0] = + SubscriptionExecutorServiceManager.submitAutoCommitWorker( + () -> { + if (isClosed()) { + if (Objects.nonNull(future[0])) { + future[0].cancel(false); + LOGGER.info("SubscriptionPullConsumer {} cancel auto commit worker", this); + } + return; + } + new AutoCommitWorker().run(); + }, + autoCommitIntervalMs); + LOGGER.info("SubscriptionPullConsumer {} submit auto commit worker", this); + } + + private class AutoCommitWorker implements Runnable { + @Override + public void run() { + if (isClosed()) { + return; + } + + final long currentTimestamp = System.currentTimeMillis(); + long index = currentTimestamp / autoCommitIntervalMs; + if (currentTimestamp % autoCommitIntervalMs == 0) { + index -= 1; + } + + for (final Map.Entry> entry : + uncommittedMessages.headMap(index).entrySet()) { + try { + ack(entry.getValue()); + uncommittedMessages.remove(entry.getKey()); + } catch (final Exception e) { + LOGGER.warn("something unexpected happened when auto commit messages...", e); + } + } + } + } + + private void commitAllUncommittedMessages() { + for (final Map.Entry> entry : uncommittedMessages.entrySet()) { + try { + ack(entry.getValue()); + uncommittedMessages.remove(entry.getKey()); + } catch (final Exception e) { + LOGGER.warn("something unexpected happened when commit messages during close", e); + } + } + } + + /////////////////////////////// stringify /////////////////////////////// + + @Override + public String toString() { + return "SubscriptionPullConsumer" + this.coreReportMessage(); + } + + @Override + protected Map coreReportMessage() { + final Map coreReportMessage = super.coreReportMessage(); + coreReportMessage.put("autoCommit", String.valueOf(autoCommit)); + return coreReportMessage; + } + + @Override + protected Map allReportMessage() { + final Map allReportMessage = super.allReportMessage(); + allReportMessage.put("autoCommit", String.valueOf(autoCommit)); + allReportMessage.put("autoCommitIntervalMs", String.valueOf(autoCommitIntervalMs)); + if (autoCommit) { + allReportMessage.put("uncommittedMessages", uncommittedMessages.toString()); + } + return allReportMessage; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumerBuilder.java new file mode 100644 index 0000000000000..3d9561adab27f --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumerBuilder.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; + +import java.util.List; + +public class AbstractSubscriptionPullConsumerBuilder extends AbstractSubscriptionConsumerBuilder { + + protected boolean autoCommit = ConsumerConstant.AUTO_COMMIT_DEFAULT_VALUE; + protected long autoCommitIntervalMs = ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_DEFAULT_VALUE; + + @Override + public AbstractSubscriptionPullConsumerBuilder host(final String host) { + super.host(host); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder port(final Integer port) { + super.port(port); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder nodeUrls(final List nodeUrls) { + super.nodeUrls(nodeUrls); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder username(final String username) { + super.username(username); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder password(final String password) { + super.password(password); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder consumerId(final String consumerId) { + super.consumerId(consumerId); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder consumerGroupId(final String consumerGroupId) { + super.consumerGroupId(consumerGroupId); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder heartbeatIntervalMs( + final long heartbeatIntervalMs) { + super.heartbeatIntervalMs(heartbeatIntervalMs); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder fileSaveDir(final String fileSaveDir) { + super.fileSaveDir(fileSaveDir); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + super.fileSaveFsync(fileSaveFsync); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize(thriftMaxFrameSize); + return this; + } + + @Override + public AbstractSubscriptionPullConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + super.maxPollParallelism(maxPollParallelism); + return this; + } + + public AbstractSubscriptionPullConsumerBuilder autoCommit(final boolean autoCommit) { + this.autoCommit = autoCommit; + return this; + } + + public AbstractSubscriptionPullConsumerBuilder autoCommitIntervalMs( + final long autoCommitIntervalMs) { + this.autoCommitIntervalMs = + Math.max(autoCommitIntervalMs, ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_MIN_VALUE); + return this; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java new file mode 100644 index 0000000000000..3ff93db218b27 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeListener; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.util.CollectionUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The {@link AbstractSubscriptionPushConsumer} corresponds to the push consumption mode in the + * message queue. + * + *

User code is triggered by newly arrived data events and only needs to pre-configure message + * acknowledgment strategy ({@link #ackStrategy}) and consumption handling logic ({@link + * #consumeListener}). + * + *

User code does not need to manually commit the consumption progress. + */ +public abstract class AbstractSubscriptionPushConsumer extends AbstractSubscriptionConsumer { + + private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionTreePushConsumer.class); + + private final AckStrategy ackStrategy; + private final ConsumeListener consumeListener; + + // avoid interval less than or equal to zero + private final long autoPollIntervalMs; + private final long autoPollTimeoutMs; + + private final AtomicBoolean isClosed = new AtomicBoolean(true); + + protected AbstractSubscriptionPushConsumer( + final AbstractSubscriptionPushConsumerBuilder builder) { + super(builder); + + this.ackStrategy = builder.ackStrategy; + this.consumeListener = builder.consumeListener; + + this.autoPollIntervalMs = builder.autoPollIntervalMs; + this.autoPollTimeoutMs = builder.autoPollTimeoutMs; + } + + public AbstractSubscriptionPushConsumer(final Properties config) { + this( + config, + (AckStrategy) + config.getOrDefault(ConsumerConstant.ACK_STRATEGY_KEY, AckStrategy.defaultValue()), + (ConsumeListener) + config.getOrDefault( + ConsumerConstant.CONSUME_LISTENER_KEY, + (ConsumeListener) message -> ConsumeResult.SUCCESS), + (Long) + config.getOrDefault( + ConsumerConstant.AUTO_POLL_INTERVAL_MS_KEY, + ConsumerConstant.AUTO_POLL_INTERVAL_MS_DEFAULT_VALUE), + (Long) + config.getOrDefault( + ConsumerConstant.AUTO_POLL_TIMEOUT_MS_KEY, + ConsumerConstant.AUTO_POLL_TIMEOUT_MS_DEFAULT_VALUE)); + } + + protected AbstractSubscriptionPushConsumer( + final Properties config, + final AckStrategy ackStrategy, + final ConsumeListener consumeListener, + final long autoPollIntervalMs, + final long autoPollTimeoutMs) { + super( + new AbstractSubscriptionPushConsumerBuilder() + .ackStrategy(ackStrategy) + .consumeListener(consumeListener) + .autoPollIntervalMs(autoPollIntervalMs) + .autoPollTimeoutMs(autoPollTimeoutMs), + config); + + this.ackStrategy = ackStrategy; + this.consumeListener = consumeListener; + + // avoid interval less than or equal to zero + this.autoPollIntervalMs = Math.max(autoPollIntervalMs, 1); + this.autoPollTimeoutMs = + Math.max(autoPollTimeoutMs, ConsumerConstant.AUTO_POLL_TIMEOUT_MS_MIN_VALUE); + } + + /////////////////////////////// open & close /////////////////////////////// + + protected synchronized void open() throws SubscriptionException { + if (!isClosed.get()) { + return; + } + + super.open(); + + // set isClosed to false before submitting workers + isClosed.set(false); + + // submit auto poll worker + submitAutoPollWorker(); + } + + @Override + public synchronized void close() { + if (isClosed.get()) { + return; + } + + super.close(); + isClosed.set(true); + } + + @Override + boolean isClosed() { + return isClosed.get(); + } + + /////////////////////////////// auto poll /////////////////////////////// + + private void submitAutoPollWorker() { + final ScheduledFuture[] future = new ScheduledFuture[1]; + future[0] = + SubscriptionExecutorServiceManager.submitAutoPollWorker( + () -> { + if (isClosed()) { + if (Objects.nonNull(future[0])) { + future[0].cancel(false); + LOGGER.info("SubscriptionPushConsumer {} cancel auto poll worker", this); + } + return; + } + new AutoPollWorker().run(); + }, + autoPollIntervalMs); + LOGGER.info("SubscriptionPushConsumer {} submit auto poll worker", this); + } + + class AutoPollWorker implements Runnable { + @Override + public void run() { + if (isClosed()) { + return; + } + + if (subscribedTopics.isEmpty()) { + return; + } + + try { + final List messages = + multiplePoll(subscribedTopics.keySet(), autoPollTimeoutMs); + if (messages.isEmpty()) { + LOGGER.info( + "SubscriptionPushConsumer {} poll empty message from topics {} after {} millisecond(s)", + this, + CollectionUtils.getLimitedString(subscribedTopics.keySet(), 32), + autoPollTimeoutMs); + return; + } + + if (ackStrategy.equals(AckStrategy.BEFORE_CONSUME)) { + ack(messages); + } + + final List messagesToAck = new ArrayList<>(); + final List messagesToNack = new ArrayList<>(); + for (final SubscriptionMessage message : messages) { + final ConsumeResult consumeResult; + try { + consumeResult = consumeListener.onReceive(message); + if (Objects.equals(ConsumeResult.SUCCESS, consumeResult)) { + messagesToAck.add(message); + } else { + LOGGER.warn("Consumer listener result failure when consuming message: {}", message); + messagesToNack.add(message); + } + } catch (final Exception e) { + LOGGER.warn( + "Consumer listener raised an exception while consuming message: {}", message, e); + messagesToNack.add(message); + } + } + + if (ackStrategy.equals(AckStrategy.AFTER_CONSUME)) { + ack(messagesToAck); + nack(messagesToNack); + } + } catch (final Exception e) { + LOGGER.warn("something unexpected happened when auto poll messages...", e); + } + } + } + + /////////////////////////////// stringify /////////////////////////////// + + @Override + public String toString() { + return "SubscriptionPushConsumer" + this.coreReportMessage(); + } + + @Override + protected Map coreReportMessage() { + final Map coreReportMessage = super.coreReportMessage(); + coreReportMessage.put("ackStrategy", ackStrategy.toString()); + return coreReportMessage; + } + + @Override + protected Map allReportMessage() { + final Map allReportMessage = super.allReportMessage(); + allReportMessage.put("ackStrategy", ackStrategy.toString()); + allReportMessage.put("autoPollIntervalMs", String.valueOf(autoPollIntervalMs)); + allReportMessage.put("autoPollTimeoutMs", String.valueOf(autoPollTimeoutMs)); + return allReportMessage; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumerBuilder.java new file mode 100644 index 0000000000000..9dba1c3989713 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumerBuilder.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeListener; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; + +import java.util.List; + +public class AbstractSubscriptionPushConsumerBuilder extends AbstractSubscriptionConsumerBuilder { + + protected AckStrategy ackStrategy = AckStrategy.defaultValue(); + protected ConsumeListener consumeListener = message -> ConsumeResult.SUCCESS; + + protected long autoPollIntervalMs = ConsumerConstant.AUTO_POLL_INTERVAL_MS_DEFAULT_VALUE; + protected long autoPollTimeoutMs = ConsumerConstant.AUTO_POLL_TIMEOUT_MS_DEFAULT_VALUE; + + @Override + public AbstractSubscriptionPushConsumerBuilder host(final String host) { + super.host(host); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder port(final Integer port) { + super.port(port); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder nodeUrls(final List nodeUrls) { + super.nodeUrls(nodeUrls); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder username(final String username) { + super.username(username); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder password(final String password) { + super.password(password); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder consumerId(final String consumerId) { + super.consumerId(consumerId); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder consumerGroupId(final String consumerGroupId) { + super.consumerGroupId(consumerGroupId); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder heartbeatIntervalMs( + final long heartbeatIntervalMs) { + super.heartbeatIntervalMs(heartbeatIntervalMs); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder fileSaveDir(final String fileSaveDir) { + super.fileSaveDir(fileSaveDir); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + super.fileSaveFsync(fileSaveFsync); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize(thriftMaxFrameSize); + return this; + } + + @Override + public AbstractSubscriptionPushConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + super.maxPollParallelism(maxPollParallelism); + return this; + } + + public AbstractSubscriptionPushConsumerBuilder ackStrategy(final AckStrategy ackStrategy) { + this.ackStrategy = ackStrategy; + return this; + } + + public AbstractSubscriptionPushConsumerBuilder consumeListener( + final ConsumeListener consumeListener) { + this.consumeListener = consumeListener; + return this; + } + + public AbstractSubscriptionPushConsumerBuilder autoPollIntervalMs(final long autoPollIntervalMs) { + // avoid interval less than or equal to zero + this.autoPollIntervalMs = Math.max(autoPollIntervalMs, 1); + return this; + } + + public AbstractSubscriptionPushConsumerBuilder autoPollTimeoutMs(final long autoPollTimeoutMs) { + this.autoPollTimeoutMs = + Math.max(autoPollTimeoutMs, ConsumerConstant.AUTO_POLL_TIMEOUT_MS_MIN_VALUE); + return this; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionExecutorServiceManager.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionExecutorServiceManager.java new file mode 100644 index 0000000000000..e31ae3ba5da7b --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionExecutorServiceManager.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.base; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public final class SubscriptionExecutorServiceManager { + + private static final Logger LOGGER = + LoggerFactory.getLogger(SubscriptionExecutorServiceManager.class); + + private static final long AWAIT_TERMINATION_TIMEOUT_MS = 15_000L; + + private static final String CONTROL_FLOW_EXECUTOR_NAME = "SubscriptionControlFlowExecutor"; + private static final String UPSTREAM_DATA_FLOW_EXECUTOR_NAME = + "SubscriptionUpstreamDataFlowExecutor"; + private static final String DOWNSTREAM_DATA_FLOW_EXECUTOR_NAME = + "SubscriptionDownstreamDataFlowExecutor"; + + /** Control Flow Executor: execute heartbeat worker, endpoints syncer and auto poll worker */ + private static final SubscriptionScheduledExecutorService CONTROL_FLOW_EXECUTOR = + new SubscriptionScheduledExecutorService( + CONTROL_FLOW_EXECUTOR_NAME, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)); + + /** Upstream Data Flow Executor: execute auto commit worker and async commit worker */ + private static final SubscriptionScheduledExecutorService UPSTREAM_DATA_FLOW_EXECUTOR = + new SubscriptionScheduledExecutorService( + UPSTREAM_DATA_FLOW_EXECUTOR_NAME, + Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)); + + /** Downstream Data Flow Executor: execute poll task */ + private static final SubscriptionExecutorService DOWNSTREAM_DATA_FLOW_EXECUTOR = + new SubscriptionExecutorService( + DOWNSTREAM_DATA_FLOW_EXECUTOR_NAME, + Math.max(Runtime.getRuntime().availableProcessors(), 1)); + + /////////////////////////////// set core pool size /////////////////////////////// + + public static void setControlFlowExecutorCorePoolSize(final int corePoolSize) { + CONTROL_FLOW_EXECUTOR.setCorePoolSize(corePoolSize); + } + + public static void setUpstreamDataFlowExecutorCorePoolSize(final int corePoolSize) { + UPSTREAM_DATA_FLOW_EXECUTOR.setCorePoolSize(corePoolSize); + } + + public static void setDownstreamDataFlowExecutorCorePoolSize(final int corePoolSize) { + DOWNSTREAM_DATA_FLOW_EXECUTOR.setCorePoolSize(corePoolSize); + } + + /////////////////////////////// shutdown hook /////////////////////////////// + + static { + // register shutdown hook + Runtime.getRuntime() + .addShutdownHook( + new Thread( + new SubscriptionExecutorServiceShutdownHook(), + "SubscriptionExecutorServiceShutdownHook")); + } + + private static class SubscriptionExecutorServiceShutdownHook implements Runnable { + + @Override + public void run() { + // shutdown executors + CONTROL_FLOW_EXECUTOR.shutdown(); + UPSTREAM_DATA_FLOW_EXECUTOR.shutdown(); + DOWNSTREAM_DATA_FLOW_EXECUTOR.shutdown(); + } + } + + /////////////////////////////// submitter /////////////////////////////// + + @SuppressWarnings("unsafeThreadSchedule") + public static ScheduledFuture submitHeartbeatWorker( + final Runnable task, final long heartbeatIntervalMs) { + CONTROL_FLOW_EXECUTOR.launchIfNeeded(); + return CONTROL_FLOW_EXECUTOR.scheduleWithFixedDelay( + task, + generateRandomInitialDelayMs(heartbeatIntervalMs), + heartbeatIntervalMs, + TimeUnit.MILLISECONDS); + } + + @SuppressWarnings("unsafeThreadSchedule") + public static ScheduledFuture submitEndpointsSyncer( + final Runnable task, final long endpointsSyncIntervalMs) { + CONTROL_FLOW_EXECUTOR.launchIfNeeded(); + return CONTROL_FLOW_EXECUTOR.scheduleWithFixedDelay( + task, + generateRandomInitialDelayMs(endpointsSyncIntervalMs), + endpointsSyncIntervalMs, + TimeUnit.MILLISECONDS); + } + + @SuppressWarnings("unsafeThreadSchedule") + public static ScheduledFuture submitAutoPollWorker( + final Runnable task, final long autoPollIntervalMs) { + CONTROL_FLOW_EXECUTOR.launchIfNeeded(); + return CONTROL_FLOW_EXECUTOR.scheduleWithFixedDelay( + task, + generateRandomInitialDelayMs(autoPollIntervalMs), + autoPollIntervalMs, + TimeUnit.MILLISECONDS); + } + + @SuppressWarnings("unsafeThreadSchedule") + public static ScheduledFuture submitAutoCommitWorker( + final Runnable task, final long autoCommitIntervalMs) { + UPSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); + return UPSTREAM_DATA_FLOW_EXECUTOR.scheduleWithFixedDelay( + task, + generateRandomInitialDelayMs(autoCommitIntervalMs), + autoCommitIntervalMs, + TimeUnit.MILLISECONDS); + } + + public static void submitAsyncCommitWorker(final Runnable task) { + UPSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); + UPSTREAM_DATA_FLOW_EXECUTOR.submit(task); + } + + public static List> submitMultiplePollTasks( + final Collection> tasks, final long timeoutMs) + throws InterruptedException { + DOWNSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); + return DOWNSTREAM_DATA_FLOW_EXECUTOR.invokeAll(tasks, timeoutMs); + } + + public static int getAvailableThreadCountForPollTasks() { + DOWNSTREAM_DATA_FLOW_EXECUTOR.launchIfNeeded(); + return DOWNSTREAM_DATA_FLOW_EXECUTOR.getAvailableCount(); + } + + /////////////////////////////// subscription executor service /////////////////////////////// + + private static class SubscriptionExecutorService { + + String name; + volatile int corePoolSize; + volatile ExecutorService executor; + + SubscriptionExecutorService(final String name, final int corePoolSize) { + this.name = name; + this.corePoolSize = corePoolSize; + } + + boolean isShutdown() { + return Objects.isNull(this.executor); + } + + void setCorePoolSize(final int corePoolSize) { + if (isShutdown()) { + synchronized (this) { + if (isShutdown()) { + this.corePoolSize = corePoolSize; + return; + } + } + } + + LOGGER.warn( + "{} has been launched, set core pool size to {} will be ignored", + this.name, + corePoolSize); + } + + void launchIfNeeded() { + if (isShutdown()) { + synchronized (this) { + if (isShutdown()) { + LOGGER.info("Launching {} with core pool size {}...", this.name, this.corePoolSize); + + this.executor = + Executors.newFixedThreadPool( + this.corePoolSize, + r -> { + final Thread t = + new Thread(Thread.currentThread().getThreadGroup(), r, this.name, 0); + if (!t.isDaemon()) { + t.setDaemon(true); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + }); + } + } + } + } + + void shutdown() { + if (!isShutdown()) { + synchronized (this) { + if (!isShutdown()) { + LOGGER.info("Shutting down {}...", this.name); + + this.executor.shutdown(); + try { + if (!this.executor.awaitTermination( + AWAIT_TERMINATION_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + this.executor.shutdownNow(); + LOGGER.warn( + "Interrupt the worker, which may cause some task inconsistent. Please check the biz logs."); + if (!this.executor.awaitTermination( + AWAIT_TERMINATION_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + LOGGER.error( + "Thread pool can't be shutdown even with interrupting worker threads, which may cause some task inconsistent. Please check the biz logs."); + } + } + } catch (final InterruptedException e) { + this.executor.shutdownNow(); + LOGGER.error( + "The current thread is interrupted when it is trying to stop the worker threads. This may leave an inconsistent state. Please check the biz logs."); + Thread.currentThread().interrupt(); + } + + this.executor = null; + } + } + } + } + + Future submit(final Runnable task) { + if (!isShutdown()) { + synchronized (this) { + if (!isShutdown()) { + return this.executor.submit(task); + } + } + } + + LOGGER.warn("{} has not been launched, ignore submit task", this.name); + return null; + } + + List> invokeAll( + final Collection> tasks, final long timeoutMs) + throws InterruptedException { + if (!isShutdown()) { + synchronized (this) { + if (!isShutdown()) { + return this.executor.invokeAll(tasks, timeoutMs, TimeUnit.MILLISECONDS); + } + } + } + + LOGGER.warn("{} has not been launched, ignore invoke all tasks", this.name); + return null; + } + + int getAvailableCount() { + if (!isShutdown()) { + synchronized (this) { + if (!isShutdown()) { + // TODO: temporarily disable multiple poll + return 0; + // return Math.max( + // ((ThreadPoolExecutor) this.executor).getCorePoolSize() + // - ((ThreadPoolExecutor) this.executor).getActiveCount(), + // 0); + } + } + } + + LOGGER.warn("{} has not been launched, return zero", this.name); + return 0; + } + } + + private static class SubscriptionScheduledExecutorService extends SubscriptionExecutorService { + + SubscriptionScheduledExecutorService(final String name, final int corePoolSize) { + super(name, corePoolSize); + } + + @Override + void launchIfNeeded() { + if (isShutdown()) { + synchronized (this) { + if (isShutdown()) { + LOGGER.info("Launching {} with core pool size {}...", this.name, this.corePoolSize); + + this.executor = + Executors.newScheduledThreadPool( + this.corePoolSize, + r -> { + final Thread t = + new Thread(Thread.currentThread().getThreadGroup(), r, this.name, 0); + if (!t.isDaemon()) { + t.setDaemon(true); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + }); + } + } + } + } + + @SuppressWarnings("unsafeThreadSchedule") + ScheduledFuture scheduleWithFixedDelay( + final Runnable task, final long initialDelay, final long delay, final TimeUnit unit) { + if (!isShutdown()) { + synchronized (this) { + if (!isShutdown()) { + return ((ScheduledExecutorService) this.executor) + .scheduleWithFixedDelay(task, initialDelay, delay, unit); + } + } + } + + LOGGER.warn("{} has not been launched, ignore scheduleWithFixedDelay for task", this.name); + return null; + } + } + + /////////////////////////////// utility /////////////////////////////// + + private static long generateRandomInitialDelayMs(final long maxMs) { + return (long) (Math.random() * maxMs); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTableProvider.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTableProvider.java new file mode 100644 index 0000000000000..40492876b887b --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTableProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.table; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.session.AbstractSessionBuilder; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider; + +final class SubscriptionTableProvider extends AbstractSubscriptionProvider { + + SubscriptionTableProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + super(endPoint, username, password, consumerId, consumerGroupId, thriftMaxFrameSize); + } + + @Override + protected AbstractSessionBuilder constructSubscriptionSessionBuilder( + final String host, + final int port, + final String username, + final String password, + final int thriftMaxFrameSize) { + return new SubscriptionTableSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .thriftMaxFrameSize(thriftMaxFrameSize); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java new file mode 100644 index 0000000000000..9e51f7438ff01 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.table; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class SubscriptionTablePullConsumer extends AbstractSubscriptionPullConsumer + implements ISubscriptionTablePullConsumer { + + /////////////////////////////// provider /////////////////////////////// + + @Override + protected AbstractSubscriptionProvider constructSubscriptionProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + return new SubscriptionTableProvider( + endPoint, username, password, consumerId, consumerGroupId, thriftMaxFrameSize); + } + + /////////////////////////////// ctor /////////////////////////////// + + protected SubscriptionTablePullConsumer(final SubscriptionTablePullConsumerBuilder builder) { + super(builder); + } + + /////////////////////////////// interface /////////////////////////////// + + @Override + public void open() throws SubscriptionException { + super.open(); + } + + @Override + public void close() throws SubscriptionException { + super.close(); + } + + @Override + public List poll(final Duration timeout) throws SubscriptionException { + return super.poll(timeout); + } + + @Override + public List poll(final long timeoutMs) throws SubscriptionException { + return super.poll(timeoutMs); + } + + @Override + public List poll(final Set topicNames, final Duration timeout) + throws SubscriptionException { + return super.poll(topicNames, timeout); + } + + @Override + public List poll(final Set topicNames, final long timeoutMs) { + return super.poll(topicNames, timeoutMs); + } + + @Override + public void subscribe(final String topicName) throws SubscriptionException { + super.subscribe(topicName); + } + + @Override + public void subscribe(final String... topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void subscribe(final Set topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void unsubscribe(final String topicName) throws SubscriptionException { + super.unsubscribe(topicName); + } + + @Override + public void unsubscribe(final String... topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public void unsubscribe(final Set topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public void commitSync(final SubscriptionMessage message) throws SubscriptionException { + super.commitSync(message); + } + + @Override + public void commitSync(final Iterable messages) + throws SubscriptionException { + super.commitSync(messages); + } + + @Override + public CompletableFuture commitAsync(final SubscriptionMessage message) { + return super.commitAsync(message); + } + + @Override + public CompletableFuture commitAsync(final Iterable messages) { + return super.commitAsync(messages); + } + + @Override + public void commitAsync(final SubscriptionMessage message, final AsyncCommitCallback callback) { + super.commitAsync(message, callback); + } + + @Override + public void commitAsync( + final Iterable messages, final AsyncCommitCallback callback) { + super.commitAsync(messages, callback); + } + + @Override + public String getConsumerId() { + return super.getConsumerId(); + } + + @Override + public String getConsumerGroupId() { + return super.getConsumerGroupId(); + } + + @Override + public boolean allTopicMessagesHaveBeenConsumed() { + return super.allTopicMessagesHaveBeenConsumed(); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumerBuilder.java new file mode 100644 index 0000000000000..b85b669876a88 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumerBuilder.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.table; + +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPullConsumerBuilder; + +import java.util.List; + +public class SubscriptionTablePullConsumerBuilder extends AbstractSubscriptionPullConsumerBuilder { + + @Override + public SubscriptionTablePullConsumerBuilder host(final String host) { + super.host(host); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder port(final Integer port) { + super.port(port); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder nodeUrls(final List nodeUrls) { + super.nodeUrls(nodeUrls); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder username(final String username) { + super.username(username); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder password(final String password) { + super.password(password); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder consumerId(final String consumerId) { + super.consumerId(consumerId); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder consumerGroupId(final String consumerGroupId) { + super.consumerGroupId(consumerGroupId); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder heartbeatIntervalMs(final long heartbeatIntervalMs) { + super.heartbeatIntervalMs(heartbeatIntervalMs); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder fileSaveDir(final String fileSaveDir) { + super.fileSaveDir(fileSaveDir); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + super.fileSaveFsync(fileSaveFsync); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize(thriftMaxFrameSize); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + super.maxPollParallelism(maxPollParallelism); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder autoCommit(final boolean autoCommit) { + super.autoCommit(autoCommit); + return this; + } + + @Override + public SubscriptionTablePullConsumerBuilder autoCommitIntervalMs( + final long autoCommitIntervalMs) { + super.autoCommitIntervalMs(autoCommitIntervalMs); + return this; + } + + public ISubscriptionTablePullConsumer build() { + return new SubscriptionTablePullConsumer(this); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePushConsumer.java new file mode 100644 index 0000000000000..4fc2c352af841 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePushConsumer.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.table; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePushConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPushConsumerBuilder; + +import java.util.Set; + +public class SubscriptionTablePushConsumer extends AbstractSubscriptionPushConsumer + implements ISubscriptionTablePushConsumer { + + /////////////////////////////// provider /////////////////////////////// + + @Override + protected AbstractSubscriptionProvider constructSubscriptionProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + return new SubscriptionTableProvider( + endPoint, username, password, consumerId, consumerGroupId, thriftMaxFrameSize); + } + + /////////////////////////////// ctor /////////////////////////////// + + protected SubscriptionTablePushConsumer(final AbstractSubscriptionPushConsumerBuilder builder) { + super(builder); + } + + /////////////////////////////// interface /////////////////////////////// + + @Override + public void open() throws SubscriptionException { + super.open(); + } + + @Override + public void close() throws SubscriptionException { + super.close(); + } + + @Override + public void subscribe(final String topicName) throws SubscriptionException { + super.subscribe(topicName); + } + + @Override + public void subscribe(final String... topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void subscribe(final Set topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void unsubscribe(final String topicName) throws SubscriptionException { + super.unsubscribe(topicName); + } + + @Override + public void unsubscribe(final String... topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public void unsubscribe(final Set topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public String getConsumerId() { + return super.getConsumerId(); + } + + @Override + public String getConsumerGroupId() { + return super.getConsumerGroupId(); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePushConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePushConsumerBuilder.java new file mode 100644 index 0000000000000..fcd62b235e2dc --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePushConsumerBuilder.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.table; + +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeListener; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePushConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPushConsumerBuilder; + +import java.util.List; + +public class SubscriptionTablePushConsumerBuilder extends AbstractSubscriptionPushConsumerBuilder { + + @Override + public SubscriptionTablePushConsumerBuilder host(final String host) { + super.host(host); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder port(final Integer port) { + super.port(port); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder nodeUrls(final List nodeUrls) { + super.nodeUrls(nodeUrls); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder username(final String username) { + super.username(username); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder password(final String password) { + super.password(password); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder consumerId(final String consumerId) { + super.consumerId(consumerId); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder consumerGroupId(final String consumerGroupId) { + super.consumerGroupId(consumerGroupId); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder heartbeatIntervalMs(final long heartbeatIntervalMs) { + super.heartbeatIntervalMs(heartbeatIntervalMs); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder fileSaveDir(final String fileSaveDir) { + super.fileSaveDir(fileSaveDir); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + super.fileSaveFsync(fileSaveFsync); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize(thriftMaxFrameSize); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + super.maxPollParallelism(maxPollParallelism); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder ackStrategy(final AckStrategy ackStrategy) { + super.ackStrategy(ackStrategy); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder consumeListener( + final ConsumeListener consumeListener) { + super.consumeListener(consumeListener); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder autoPollIntervalMs(final long autoPollIntervalMs) { + super.autoPollIntervalMs(autoPollIntervalMs); + return this; + } + + @Override + public SubscriptionTablePushConsumerBuilder autoPollTimeoutMs(final long autoPollTimeoutMs) { + super.autoPollTimeoutMs(autoPollTimeoutMs); + return this; + } + + public ISubscriptionTablePushConsumer build() { + return new SubscriptionTablePushConsumer(this); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreeProvider.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreeProvider.java new file mode 100644 index 0000000000000..56b07667f2d2a --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreeProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.tree; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.session.AbstractSessionBuilder; +import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider; + +final class SubscriptionTreeProvider extends AbstractSubscriptionProvider { + + SubscriptionTreeProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + super(endPoint, username, password, consumerId, consumerGroupId, thriftMaxFrameSize); + } + + @Override + protected AbstractSessionBuilder constructSubscriptionSessionBuilder( + final String host, + final int port, + final String username, + final String password, + final int thriftMaxFrameSize) { + return new SubscriptionTreeSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .thriftMaxFrameSize(thriftMaxFrameSize); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java new file mode 100644 index 0000000000000..713dd601e2d83 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.tree; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; + +import org.apache.thrift.annotation.Nullable; + +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class SubscriptionTreePullConsumer extends AbstractSubscriptionPullConsumer + implements ISubscriptionTreePullConsumer { + + /////////////////////////////// provider /////////////////////////////// + + @Override + protected AbstractSubscriptionProvider constructSubscriptionProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + return new SubscriptionTreeProvider( + endPoint, username, password, consumerId, consumerGroupId, thriftMaxFrameSize); + } + + /////////////////////////////// ctor /////////////////////////////// + + protected SubscriptionTreePullConsumer(final SubscriptionTreePullConsumerBuilder builder) { + super(builder); + } + + @Deprecated // keep for forward compatibility + private SubscriptionTreePullConsumer(final SubscriptionTreePullConsumer.Builder builder) { + super( + new SubscriptionTreePullConsumerBuilder() + .host(builder.host) + .port(builder.port) + .nodeUrls(builder.nodeUrls) + .username(builder.username) + .password(builder.password) + .consumerId(builder.consumerId) + .consumerGroupId(builder.consumerGroupId) + .heartbeatIntervalMs(builder.heartbeatIntervalMs) + .endpointsSyncIntervalMs(builder.endpointsSyncIntervalMs) + .fileSaveDir(builder.fileSaveDir) + .fileSaveFsync(builder.fileSaveFsync) + .thriftMaxFrameSize(builder.thriftMaxFrameSize) + .maxPollParallelism(builder.maxPollParallelism) + .autoCommit(builder.autoCommit) + .autoCommitIntervalMs(builder.autoCommitIntervalMs)); + } + + public SubscriptionTreePullConsumer(final Properties properties) { + this( + properties, + (Boolean) + properties.getOrDefault( + ConsumerConstant.AUTO_COMMIT_KEY, ConsumerConstant.AUTO_COMMIT_DEFAULT_VALUE), + (Long) + properties.getOrDefault( + ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_KEY, + ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_DEFAULT_VALUE)); + } + + private SubscriptionTreePullConsumer( + final Properties properties, final boolean autoCommit, final long autoCommitIntervalMs) { + super(properties, autoCommit, autoCommitIntervalMs); + } + + /////////////////////////////// interface /////////////////////////////// + + @Override + public void open() throws SubscriptionException { + super.open(); + } + + @Override + public void close() throws SubscriptionException { + super.close(); + } + + @Override + public void subscribe(final String topicName) throws SubscriptionException { + super.subscribe(topicName); + } + + @Override + public void subscribe(final String... topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void subscribe(final Set topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void unsubscribe(final String topicName) throws SubscriptionException { + super.unsubscribe(topicName); + } + + @Override + public void unsubscribe(final String... topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public void unsubscribe(final Set topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public List poll(final Duration timeout) throws SubscriptionException { + return super.poll(timeout); + } + + @Override + public List poll(final long timeoutMs) throws SubscriptionException { + return super.poll(timeoutMs); + } + + @Override + public List poll(final Set topicNames, final Duration timeout) + throws SubscriptionException { + return super.poll(topicNames, timeout); + } + + @Override + public List poll(final Set topicNames, final long timeoutMs) { + return super.poll(topicNames, timeoutMs); + } + + @Override + public void commitSync(final SubscriptionMessage message) throws SubscriptionException { + super.commitSync(message); + } + + @Override + public void commitSync(final Iterable messages) + throws SubscriptionException { + super.commitSync(messages); + } + + @Override + public CompletableFuture commitAsync(final SubscriptionMessage message) { + return super.commitAsync(message); + } + + @Override + public CompletableFuture commitAsync(final Iterable messages) { + return super.commitAsync(messages); + } + + @Override + public void commitAsync(final SubscriptionMessage message, final AsyncCommitCallback callback) { + super.commitAsync(message, callback); + } + + @Override + public void commitAsync( + final Iterable messages, final AsyncCommitCallback callback) { + super.commitAsync(messages, callback); + } + + @Override + public String getConsumerId() { + return super.getConsumerId(); + } + + @Override + public String getConsumerGroupId() { + return super.getConsumerGroupId(); + } + + @Override + public boolean allTopicMessagesHaveBeenConsumed() { + return super.allTopicMessagesHaveBeenConsumed(); + } + + /////////////////////////////// builder /////////////////////////////// + + @Deprecated // keep for forward compatibility + public static class Builder { + + private String host; + private Integer port; + private List nodeUrls; + + private String username = SessionConfig.DEFAULT_USER; + private String password = SessionConfig.DEFAULT_PASSWORD; + + private String consumerId; + private String consumerGroupId; + + private long heartbeatIntervalMs = ConsumerConstant.HEARTBEAT_INTERVAL_MS_DEFAULT_VALUE; + private long endpointsSyncIntervalMs = + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_DEFAULT_VALUE; + + private String fileSaveDir = ConsumerConstant.FILE_SAVE_DIR_DEFAULT_VALUE; + private boolean fileSaveFsync = ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE; + + private int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; + private int maxPollParallelism = ConsumerConstant.MAX_POLL_PARALLELISM_DEFAULT_VALUE; + + private boolean autoCommit = ConsumerConstant.AUTO_COMMIT_DEFAULT_VALUE; + private long autoCommitIntervalMs = ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_DEFAULT_VALUE; + + public Builder host(final String host) { + this.host = host; + return this; + } + + public Builder port(final Integer port) { + this.port = port; + return this; + } + + public Builder nodeUrls(final List nodeUrls) { + this.nodeUrls = nodeUrls; + return this; + } + + public Builder username(final String username) { + this.username = username; + return this; + } + + public Builder password(final String password) { + this.password = password; + return this; + } + + public Builder consumerId(@Nullable final String consumerId) { + if (Objects.isNull(consumerId)) { + return this; + } + this.consumerId = IdentifierUtils.checkAndParseIdentifier(consumerId); + return this; + } + + public Builder consumerGroupId(@Nullable final String consumerGroupId) { + if (Objects.isNull(consumerGroupId)) { + return this; + } + this.consumerGroupId = IdentifierUtils.checkAndParseIdentifier(consumerGroupId); + return this; + } + + public Builder heartbeatIntervalMs(final long heartbeatIntervalMs) { + this.heartbeatIntervalMs = + Math.max(heartbeatIntervalMs, ConsumerConstant.HEARTBEAT_INTERVAL_MS_MIN_VALUE); + return this; + } + + public Builder endpointsSyncIntervalMs(final long endpointsSyncIntervalMs) { + this.endpointsSyncIntervalMs = + Math.max(endpointsSyncIntervalMs, ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE); + return this; + } + + public Builder fileSaveDir(final String fileSaveDir) { + this.fileSaveDir = fileSaveDir; + return this; + } + + public Builder fileSaveFsync(final boolean fileSaveFsync) { + this.fileSaveFsync = fileSaveFsync; + return this; + } + + public Builder thriftMaxFrameSize(final int thriftMaxFrameSize) { + this.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + public Builder maxPollParallelism(final int maxPollParallelism) { + // Here the minimum value of max poll parallelism is set to 1 instead of 0, in order to use a + // single thread to execute poll whenever there are idle resources available, thereby + // achieving strict timeout. + this.maxPollParallelism = Math.max(maxPollParallelism, 1); + return this; + } + + public Builder autoCommit(final boolean autoCommit) { + this.autoCommit = autoCommit; + return this; + } + + public Builder autoCommitIntervalMs(final long autoCommitIntervalMs) { + this.autoCommitIntervalMs = + Math.max(autoCommitIntervalMs, ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_MIN_VALUE); + return this; + } + + public SubscriptionTreePullConsumer buildPullConsumer() { + return new SubscriptionTreePullConsumer(this); + } + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumerBuilder.java new file mode 100644 index 0000000000000..f3d3b7afba82e --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumerBuilder.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.tree; + +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTreePullConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPullConsumerBuilder; + +import java.util.List; + +public class SubscriptionTreePullConsumerBuilder extends AbstractSubscriptionPullConsumerBuilder { + + @Override + public SubscriptionTreePullConsumerBuilder host(final String host) { + super.host(host); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder port(final Integer port) { + super.port(port); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder nodeUrls(final List nodeUrls) { + super.nodeUrls(nodeUrls); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder username(final String username) { + super.username(username); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder password(final String password) { + super.password(password); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder consumerId(final String consumerId) { + super.consumerId(consumerId); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder consumerGroupId(final String consumerGroupId) { + super.consumerGroupId(consumerGroupId); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder heartbeatIntervalMs(final long heartbeatIntervalMs) { + super.heartbeatIntervalMs(heartbeatIntervalMs); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder fileSaveDir(final String fileSaveDir) { + super.fileSaveDir(fileSaveDir); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + super.fileSaveFsync(fileSaveFsync); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize(thriftMaxFrameSize); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + super.maxPollParallelism(maxPollParallelism); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder autoCommit(final boolean autoCommit) { + super.autoCommit(autoCommit); + return this; + } + + @Override + public SubscriptionTreePullConsumerBuilder autoCommitIntervalMs(final long autoCommitIntervalMs) { + super.autoCommitIntervalMs(autoCommitIntervalMs); + return this; + } + + public ISubscriptionTreePullConsumer build() { + return new SubscriptionTreePullConsumer(this); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePushConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePushConsumer.java new file mode 100644 index 0000000000000..cd5c548121e2d --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePushConsumer.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.tree; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeListener; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.util.IdentifierUtils; + +import org.apache.thrift.annotation.Nullable; + +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; + +public class SubscriptionTreePushConsumer extends AbstractSubscriptionPushConsumer + implements ISubscriptionTreePushConsumer { + + /////////////////////////////// provider /////////////////////////////// + + @Override + protected AbstractSubscriptionProvider constructSubscriptionProvider( + final TEndPoint endPoint, + final String username, + final String password, + final String consumerId, + final String consumerGroupId, + final int thriftMaxFrameSize) { + return new SubscriptionTreeProvider( + endPoint, username, password, consumerId, consumerGroupId, thriftMaxFrameSize); + } + + /////////////////////////////// ctor /////////////////////////////// + + public SubscriptionTreePushConsumer(final SubscriptionTreePushConsumerBuilder builder) { + super(builder); + } + + @Deprecated // keep for forward compatibility + private SubscriptionTreePushConsumer(final Builder builder) { + super( + new SubscriptionTreePushConsumerBuilder() + .host(builder.host) + .port(builder.port) + .nodeUrls(builder.nodeUrls) + .username(builder.username) + .password(builder.password) + .consumerId(builder.consumerId) + .consumerGroupId(builder.consumerGroupId) + .heartbeatIntervalMs(builder.heartbeatIntervalMs) + .endpointsSyncIntervalMs(builder.endpointsSyncIntervalMs) + .fileSaveDir(builder.fileSaveDir) + .fileSaveFsync(builder.fileSaveFsync) + .thriftMaxFrameSize(builder.thriftMaxFrameSize) + .maxPollParallelism(builder.maxPollParallelism) + .ackStrategy(builder.ackStrategy) + .consumeListener(builder.consumeListener) + .autoPollIntervalMs(builder.autoPollIntervalMs) + .autoPollTimeoutMs(builder.autoPollTimeoutMs)); + } + + public SubscriptionTreePushConsumer(final Properties config) { + this( + config, + (AckStrategy) + config.getOrDefault(ConsumerConstant.ACK_STRATEGY_KEY, AckStrategy.defaultValue()), + (ConsumeListener) + config.getOrDefault( + ConsumerConstant.CONSUME_LISTENER_KEY, + (ConsumeListener) message -> ConsumeResult.SUCCESS), + (Long) + config.getOrDefault( + ConsumerConstant.AUTO_POLL_INTERVAL_MS_KEY, + ConsumerConstant.AUTO_POLL_INTERVAL_MS_DEFAULT_VALUE), + (Long) + config.getOrDefault( + ConsumerConstant.AUTO_POLL_TIMEOUT_MS_KEY, + ConsumerConstant.AUTO_POLL_TIMEOUT_MS_DEFAULT_VALUE)); + } + + private SubscriptionTreePushConsumer( + final Properties config, + final AckStrategy ackStrategy, + final ConsumeListener consumeListener, + final long autoPollIntervalMs, + final long autoPollTimeoutMs) { + super(config, ackStrategy, consumeListener, autoPollIntervalMs, autoPollTimeoutMs); + } + + /////////////////////////////// interface /////////////////////////////// + + @Override + public void open() throws SubscriptionException { + super.open(); + } + + @Override + public void close() throws SubscriptionException { + super.close(); + } + + @Override + public void subscribe(final String topicName) throws SubscriptionException { + super.subscribe(topicName); + } + + @Override + public void subscribe(final String... topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void subscribe(final Set topicNames) throws SubscriptionException { + super.subscribe(topicNames); + } + + @Override + public void unsubscribe(final String topicName) throws SubscriptionException { + super.unsubscribe(topicName); + } + + @Override + public void unsubscribe(final String... topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public void unsubscribe(final Set topicNames) throws SubscriptionException { + super.unsubscribe(topicNames); + } + + @Override + public String getConsumerId() { + return super.getConsumerId(); + } + + @Override + public String getConsumerGroupId() { + return super.getConsumerGroupId(); + } + + /////////////////////////////// builder /////////////////////////////// + + @Deprecated // keep for forward compatibility + public static class Builder { + + private String host; + private Integer port; + private List nodeUrls; + + private String username = SessionConfig.DEFAULT_USER; + private String password = SessionConfig.DEFAULT_PASSWORD; + + private String consumerId; + private String consumerGroupId; + + private long heartbeatIntervalMs = ConsumerConstant.HEARTBEAT_INTERVAL_MS_DEFAULT_VALUE; + private long endpointsSyncIntervalMs = + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_DEFAULT_VALUE; + + private String fileSaveDir = ConsumerConstant.FILE_SAVE_DIR_DEFAULT_VALUE; + private boolean fileSaveFsync = ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE; + + private int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; + private int maxPollParallelism = ConsumerConstant.MAX_POLL_PARALLELISM_DEFAULT_VALUE; + + private AckStrategy ackStrategy = AckStrategy.defaultValue(); + private ConsumeListener consumeListener = message -> ConsumeResult.SUCCESS; + + private long autoPollIntervalMs = ConsumerConstant.AUTO_POLL_INTERVAL_MS_DEFAULT_VALUE; + private long autoPollTimeoutMs = ConsumerConstant.AUTO_POLL_TIMEOUT_MS_DEFAULT_VALUE; + + public Builder host(final String host) { + this.host = host; + return this; + } + + public Builder port(final Integer port) { + this.port = port; + return this; + } + + public Builder nodeUrls(final List nodeUrls) { + this.nodeUrls = nodeUrls; + return this; + } + + public Builder username(final String username) { + this.username = username; + return this; + } + + public Builder password(final String password) { + this.password = password; + return this; + } + + public Builder consumerId(@Nullable final String consumerId) { + if (Objects.isNull(consumerId)) { + return this; + } + this.consumerId = IdentifierUtils.checkAndParseIdentifier(consumerId); + return this; + } + + public Builder consumerGroupId(@Nullable final String consumerGroupId) { + if (Objects.isNull(consumerGroupId)) { + return this; + } + this.consumerGroupId = IdentifierUtils.checkAndParseIdentifier(consumerGroupId); + return this; + } + + public Builder heartbeatIntervalMs(final long heartbeatIntervalMs) { + this.heartbeatIntervalMs = + Math.max(heartbeatIntervalMs, ConsumerConstant.HEARTBEAT_INTERVAL_MS_MIN_VALUE); + return this; + } + + public Builder endpointsSyncIntervalMs(final long endpointsSyncIntervalMs) { + this.endpointsSyncIntervalMs = + Math.max(endpointsSyncIntervalMs, ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE); + return this; + } + + public Builder fileSaveDir(final String fileSaveDir) { + this.fileSaveDir = fileSaveDir; + return this; + } + + public Builder fileSaveFsync(final boolean fileSaveFsync) { + this.fileSaveFsync = fileSaveFsync; + return this; + } + + public Builder thriftMaxFrameSize(final int thriftMaxFrameSize) { + this.thriftMaxFrameSize = thriftMaxFrameSize; + return this; + } + + public Builder maxPollParallelism(final int maxPollParallelism) { + // Here the minimum value of max poll parallelism is set to 1 instead of 0, in order to use a + // single thread to execute poll whenever there are idle resources available, thereby + // achieving strict timeout. + this.maxPollParallelism = Math.max(maxPollParallelism, 1); + return this; + } + + public Builder ackStrategy(final AckStrategy ackStrategy) { + this.ackStrategy = ackStrategy; + return this; + } + + public Builder consumeListener(final ConsumeListener consumeListener) { + this.consumeListener = consumeListener; + return this; + } + + public Builder autoPollIntervalMs(final long autoPollIntervalMs) { + // avoid interval less than or equal to zero + this.autoPollIntervalMs = Math.max(autoPollIntervalMs, 1); + return this; + } + + public Builder autoPollTimeoutMs(final long autoPollTimeoutMs) { + this.autoPollTimeoutMs = + Math.max(autoPollTimeoutMs, ConsumerConstant.AUTO_POLL_TIMEOUT_MS_MIN_VALUE); + return this; + } + + public SubscriptionTreePushConsumer buildPushConsumer() { + return new SubscriptionTreePushConsumer(this); + } + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePushConsumerBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePushConsumerBuilder.java new file mode 100644 index 0000000000000..44fde0ed4f0bb --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePushConsumerBuilder.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.consumer.tree; + +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeListener; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTreePushConsumer; +import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPushConsumerBuilder; + +import java.util.List; + +public class SubscriptionTreePushConsumerBuilder extends AbstractSubscriptionPushConsumerBuilder { + + @Override + public SubscriptionTreePushConsumerBuilder host(final String host) { + super.host(host); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder port(final Integer port) { + super.port(port); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder nodeUrls(final List nodeUrls) { + super.nodeUrls(nodeUrls); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder username(final String username) { + super.username(username); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder password(final String password) { + super.password(password); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder consumerId(final String consumerId) { + super.consumerId(consumerId); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder consumerGroupId(final String consumerGroupId) { + super.consumerGroupId(consumerGroupId); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder heartbeatIntervalMs(final long heartbeatIntervalMs) { + super.heartbeatIntervalMs(heartbeatIntervalMs); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder endpointsSyncIntervalMs( + final long endpointsSyncIntervalMs) { + super.endpointsSyncIntervalMs(endpointsSyncIntervalMs); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder fileSaveDir(final String fileSaveDir) { + super.fileSaveDir(fileSaveDir); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder fileSaveFsync(final boolean fileSaveFsync) { + super.fileSaveFsync(fileSaveFsync); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder thriftMaxFrameSize(final int thriftMaxFrameSize) { + super.thriftMaxFrameSize(thriftMaxFrameSize); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder maxPollParallelism(final int maxPollParallelism) { + super.maxPollParallelism(maxPollParallelism); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder ackStrategy(final AckStrategy ackStrategy) { + super.ackStrategy(ackStrategy); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder consumeListener( + final ConsumeListener consumeListener) { + super.consumeListener(consumeListener); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder autoPollIntervalMs(final long autoPollIntervalMs) { + super.autoPollIntervalMs(autoPollIntervalMs); + return this; + } + + @Override + public SubscriptionTreePushConsumerBuilder autoPollTimeoutMs(final long autoPollTimeoutMs) { + super.autoPollTimeoutMs(autoPollTimeoutMs); + return this; + } + + public ISubscriptionTreePushConsumer build() { + return new SubscriptionTreePushConsumer(this); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Subscription.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Subscription.java index e5f227be17b80..01e454cae2cc2 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Subscription.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Subscription.java @@ -21,16 +21,26 @@ public class Subscription { + private final String subscriptionId; private final String topicName; private final String consumerGroupId; private final String consumerIds; - public Subscription(String topicName, String consumerGroupId, String consumerIds) { + public Subscription( + final String subscriptionId, + final String topicName, + final String consumerGroupId, + final String consumerIds) { + this.subscriptionId = subscriptionId; this.topicName = topicName; this.consumerGroupId = consumerGroupId; this.consumerIds = consumerIds; } + public String getSubscriptionId() { + return subscriptionId; + } + public String getTopicName() { return topicName; } @@ -45,7 +55,9 @@ public String getConsumerIds() { @Override public String toString() { - return "Subscription{topicName=" + return "Subscription{subscriptionId=" + + subscriptionId + + ", topicName=" + topicName + ", consumerGroupId=" + consumerGroupId diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Topic.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Topic.java index 7b1feb97a1adf..cb9e321b5e921 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Topic.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/model/Topic.java @@ -24,7 +24,7 @@ public class Topic { private final String topicName; private final String topicAttributes; - public Topic(String topicName, String topicAttributes) { + public Topic(final String topicName, final String topicAttributes) { this.topicName = topicName; this.topicAttributes = topicAttributes; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java index 0ec121f96992a..e0fe83d6c9f1c 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java @@ -19,7 +19,7 @@ package org.apache.iotdb.session.subscription.payload; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionIncompatibleHandlerException; +import org.apache.iotdb.session.util.RetryUtils; import java.io.File; import java.io.IOException; @@ -56,8 +56,11 @@ public synchronized Path getPath() { */ public synchronized Path deleteFile() throws IOException { final Path sourcePath = getPath(); - Files.delete(sourcePath); - return sourcePath; + return RetryUtils.retryOnException( + () -> { + Files.delete(sourcePath); + return sourcePath; + }); } /** @@ -66,7 +69,7 @@ public synchronized Path deleteFile() throws IOException { * @throws IOException if an I/O error occurs */ public synchronized Path moveFile(final String target) throws IOException { - return this.moveFile(Paths.get(target)); + return RetryUtils.retryOnException(() -> this.moveFile(Paths.get(target))); } /** @@ -78,7 +81,8 @@ public synchronized Path moveFile(final Path target) throws IOException { if (!Files.exists(target.getParent())) { Files.createDirectories(target.getParent()); } - return Files.move(getPath(), target, StandardCopyOption.REPLACE_EXISTING); + return RetryUtils.retryOnException( + () -> Files.move(getPath(), target, StandardCopyOption.REPLACE_EXISTING)); } /** @@ -87,7 +91,7 @@ public synchronized Path moveFile(final Path target) throws IOException { * @throws IOException if an I/O error occurs */ public synchronized Path copyFile(final String target) throws IOException { - return this.copyFile(Paths.get(target)); + return RetryUtils.retryOnException(() -> this.copyFile(Paths.get(target))); } /** @@ -99,13 +103,12 @@ public synchronized Path copyFile(final Path target) throws IOException { if (!Files.exists(target.getParent())) { Files.createDirectories(target.getParent()); } - return Files.copy( - getPath(), target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); - } - - @Override - public SubscriptionSessionDataSetsHandler getSessionDataSetsHandler() { - throw new SubscriptionIncompatibleHandlerException( - "SubscriptionFileHandler do not support getSessionDataSetsHandler()."); + return RetryUtils.retryOnException( + () -> + Files.copy( + getPath(), + target, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES)); } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java index 40182692178b0..f48fa485f7d61 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java @@ -19,15 +19,17 @@ package org.apache.iotdb.session.subscription.payload; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionIncompatibleHandlerException; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.thrift.annotation.Nullable; import org.apache.tsfile.write.record.Tablet; import java.util.List; +import java.util.Map; import java.util.Objects; -public class SubscriptionMessage - implements Comparable, SubscriptionMessageHandler { +public class SubscriptionMessage implements Comparable { private final SubscriptionCommitContext commitContext; @@ -36,17 +38,19 @@ public class SubscriptionMessage private final SubscriptionMessageHandler handler; public SubscriptionMessage( - final SubscriptionCommitContext commitContext, final List tablets) { + final SubscriptionCommitContext commitContext, final Map> tablets) { this.commitContext = commitContext; this.messageType = SubscriptionMessageType.SESSION_DATA_SETS_HANDLER.getType(); this.handler = new SubscriptionSessionDataSetsHandler(tablets); } public SubscriptionMessage( - final SubscriptionCommitContext commitContext, final String absolutePath) { + final SubscriptionCommitContext commitContext, + final String absolutePath, + @Nullable final String databaseName) { this.commitContext = commitContext; this.messageType = SubscriptionMessageType.TS_FILE_HANDLER.getType(); - this.handler = new SubscriptionTsFileHandler(absolutePath); + this.handler = new SubscriptionTsFileHandler(absolutePath, databaseName); } public SubscriptionCommitContext getCommitContext() { @@ -94,13 +98,20 @@ public String toString() { /////////////////////////////// handlers /////////////////////////////// - @Override public SubscriptionSessionDataSetsHandler getSessionDataSetsHandler() { - return handler.getSessionDataSetsHandler(); + if (handler instanceof SubscriptionSessionDataSetsHandler) { + return (SubscriptionSessionDataSetsHandler) handler; + } + throw new SubscriptionIncompatibleHandlerException( + String.format( + "%s do not support getSessionDataSetsHandler().", handler.getClass().getSimpleName())); } - @Override public SubscriptionTsFileHandler getTsFileHandler() { - return handler.getTsFileHandler(); + if (handler instanceof SubscriptionTsFileHandler) { + return (SubscriptionTsFileHandler) handler; + } + throw new SubscriptionIncompatibleHandlerException( + String.format("%s do not support getTsFileHandler().", handler.getClass().getSimpleName())); } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageHandler.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageHandler.java index 9aa5527ea6e34..275f89c0d14a6 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageHandler.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageHandler.java @@ -19,9 +19,4 @@ package org.apache.iotdb.session.subscription.payload; -public interface SubscriptionMessageHandler { - - SubscriptionSessionDataSetsHandler getSessionDataSetsHandler(); - - SubscriptionTsFileHandler getTsFileHandler(); -} +public interface SubscriptionMessageHandler {} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java index ac54941f5151e..a3f7f8e3f26d1 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java @@ -20,38 +20,103 @@ package org.apache.iotdb.session.subscription.payload; import org.apache.iotdb.isession.ISessionDataSet; +import org.apache.iotdb.rpc.subscription.annotation.TableModel; +import org.apache.thrift.annotation.Nullable; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.IMeasurementSchema; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; import java.util.stream.Collectors; +import java.util.stream.Stream; public class SubscriptionSessionDataSet implements ISessionDataSet { private Tablet tablet; + @Nullable private final String databaseName; + public Tablet getTablet() { return tablet; } - public SubscriptionSessionDataSet(final Tablet tablet) { + public SubscriptionSessionDataSet(final Tablet tablet, @Nullable final String databaseName) { this.tablet = tablet; + this.databaseName = databaseName; generateRowIterator(); } + /////////////////////////////// table model /////////////////////////////// + + @TableModel private List columnCategoryList; + + @TableModel + public String getDatabaseName() { + return databaseName; + } + + @TableModel + public String getTableName() { + return tablet.getTableName(); + } + + @TableModel + public List getColumnCategories() { + if (Objects.nonNull(columnCategoryList)) { + return columnCategoryList; + } + + if (!isTableData()) { + return Collections.emptyList(); + } + + return columnCategoryList = + Stream.concat( + Stream.of(ColumnCategory.TIME), + tablet.getColumnTypes().stream() + .map( + columnCategory -> { + switch (columnCategory) { + case FIELD: + return ColumnCategory.FIELD; + case TAG: + return ColumnCategory.TAG; + case ATTRIBUTE: + return ColumnCategory.ATTRIBUTE; + default: + throw new IllegalArgumentException( + "Unknown column category: " + columnCategory); + } + })) + .collect(Collectors.toList()); + } + + @TableModel + public enum ColumnCategory { + TIME, + TAG, + FIELD, + ATTRIBUTE + } + + private boolean isTableData() { + return Objects.nonNull(databaseName); + } + /////////////////////////////// override /////////////////////////////// private List columnNameList; @@ -63,16 +128,17 @@ public List getColumnNames() { return columnNameList; } - columnNameList = new ArrayList<>(); - columnNameList.add("Time"); - - final String deviceId = tablet.deviceId; - final List schemas = tablet.getSchemas(); - columnNameList.addAll( - schemas.stream() - .map((schema) -> deviceId + "." + schema.getMeasurementId()) - .collect(Collectors.toList())); - return columnNameList; + List schemas = tablet.getSchemas(); + String deviceId = tablet.getDeviceId(); + return columnNameList = + isTableData() + ? Stream.concat( + Stream.of("time"), schemas.stream().map(IMeasurementSchema::getMeasurementName)) + .collect(Collectors.toList()) + : Stream.concat( + Stream.of("Time"), + schemas.stream().map(schema -> deviceId + "." + schema.getMeasurementName())) + .collect(Collectors.toList()); } @Override @@ -81,13 +147,12 @@ public List getColumnTypes() { return columnTypeList; } - columnTypeList = new ArrayList<>(); - columnTypeList.add(TSDataType.INT64.toString()); - - final List schemas = tablet.getSchemas(); - columnTypeList.addAll( - schemas.stream().map(schema -> schema.getType().toString()).collect(Collectors.toList())); - return columnTypeList; + List schemas = tablet.getSchemas(); + return columnTypeList = + Stream.concat( + Stream.of(TSDataType.INT64.toString()), + schemas.stream().map(schema -> schema.getType().toString())) + .collect(Collectors.toList()); } public boolean hasNext() { @@ -103,13 +168,16 @@ public RowRecord next() { final long timestamp = entry.getKey(); final int rowIndex = entry.getValue(); + BitMap[] bitMaps = tablet.getBitMaps(); for (int columnIndex = 0; columnIndex < columnSize; ++columnIndex) { final Field field; - if (tablet.bitMaps[columnIndex].isMarked(rowIndex)) { + if (bitMaps != null + && bitMaps[columnIndex] != null + && bitMaps[columnIndex].isMarked(rowIndex)) { field = new Field(null); } else { final TSDataType dataType = tablet.getSchemas().get(columnIndex).getType(); - field = generateFieldFromTabletValue(dataType, tablet.values[columnIndex], rowIndex); + field = generateFieldFromTabletValue(dataType, tablet.getValues()[columnIndex], rowIndex); } fields.add(field); } @@ -132,7 +200,7 @@ private int getColumnSize() { private void generateRowIterator() { // timestamp -> row index - final long[] timestamps = tablet.timestamps; + final long[] timestamps = tablet.getTimestamps(); final TreeMap timestampToRowIndex = new TreeMap<>(); final int rowSize = timestamps.length; for (int rowIndex = 0; rowIndex < rowSize; ++rowIndex) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSetsHandler.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSetsHandler.java index 9da0aa3805a98..bfe5d83d5faa5 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSetsHandler.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSetsHandler.java @@ -19,44 +19,73 @@ package org.apache.iotdb.session.subscription.payload; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionIncompatibleHandlerException; - import org.apache.tsfile.write.record.Tablet; -import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; public class SubscriptionSessionDataSetsHandler implements Iterable, SubscriptionMessageHandler { - private final List dataSets; - - private final List tablets; + private final Map> tablets; - public SubscriptionSessionDataSetsHandler(final List tablets) { - this.dataSets = new ArrayList<>(); + public SubscriptionSessionDataSetsHandler(final Map> tablets) { this.tablets = tablets; - tablets.forEach((tablet -> this.dataSets.add(new SubscriptionSessionDataSet(tablet)))); } @Override public Iterator iterator() { - return dataSets.iterator(); + return new Iterator() { + // Iterator over map entries: databaseName -> list of tablets + private final Iterator>> entryIterator = + tablets.entrySet().iterator(); + // Current databaseName + private String currentDatabase; + // Iterator over the current list of tablets + private Iterator tabletIterator = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + // Advance to next non-empty tablet list if needed + while (!tabletIterator.hasNext() && entryIterator.hasNext()) { + Map.Entry> entry = entryIterator.next(); + currentDatabase = entry.getKey(); + List list = entry.getValue(); + tabletIterator = (list != null ? list.iterator() : Collections.emptyIterator()); + } + return tabletIterator.hasNext(); + } + + @Override + public SubscriptionSessionDataSet next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Tablet tablet = tabletIterator.next(); + return new SubscriptionSessionDataSet(tablet, currentDatabase); + } + }; } public Iterator tabletIterator() { - return tablets.iterator(); - } + return new Iterator() { + final Iterator iterator = iterator(); - @Override - public SubscriptionSessionDataSetsHandler getSessionDataSetsHandler() { - return this; - } + @Override + public boolean hasNext() { + return iterator.hasNext(); + } - @Override - public SubscriptionTsFileHandler getTsFileHandler() { - throw new SubscriptionIncompatibleHandlerException( - "SubscriptionSessionDataSetsHandler do not support getSessionDataSetsHandler()."); + @Override + public Tablet next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return iterator.next().getTablet(); + } + }; } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionTsFileHandler.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionTsFileHandler.java index c1c84c79d78f1..749f1bcb8b717 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionTsFileHandler.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionTsFileHandler.java @@ -19,6 +19,9 @@ package org.apache.iotdb.session.subscription.payload; +import org.apache.iotdb.rpc.subscription.annotation.TableModel; + +import org.apache.thrift.annotation.Nullable; import org.apache.tsfile.read.TsFileReader; import org.apache.tsfile.read.TsFileSequenceReader; @@ -26,16 +29,19 @@ public class SubscriptionTsFileHandler extends SubscriptionFileHandler { - public SubscriptionTsFileHandler(final String absolutePath) { + @Nullable private final String databaseName; + + public SubscriptionTsFileHandler(final String absolutePath, @Nullable final String databaseName) { super(absolutePath); + this.databaseName = databaseName; } public TsFileReader openReader() throws IOException { return new TsFileReader(new TsFileSequenceReader(absolutePath)); } - @Override - public SubscriptionTsFileHandler getTsFileHandler() { - return this; + @TableModel + public String getDatabaseName() { + return databaseName; } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/CollectionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/CollectionUtils.java new file mode 100644 index 0000000000000..5f2cfb5ba60ba --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/CollectionUtils.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.util; + +import java.util.Collection; +import java.util.stream.Collectors; + +public class CollectionUtils { + + public static String getLimitedString(final Collection collection, final int limit) { + return collection.stream().limit(limit).collect(Collectors.toList()) + + (collection.size() > limit ? " ... (" + (collection.size() - limit) + " more)" : ""); + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java index 9f6d09ef44e0f..6947ac9ef7b42 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/IdentifierUtils.java @@ -24,12 +24,20 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.read.common.parser.PathVisitor; +import java.util.Objects; + public class IdentifierUtils { /** * refer org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor#parseIdentifier(java.lang.String) */ - public static String parseIdentifier(final String src) { + public static String checkAndParseIdentifier(final String src) { + if (Objects.isNull(src)) { + throw new SubscriptionIdentifierSemanticException("null identifier is not supported"); + } + if (src.isEmpty()) { + throw new SubscriptionIdentifierSemanticException("empty identifier is not supported"); + } if (src.startsWith(TsFileConstant.BACK_QUOTE_STRING) && src.endsWith(TsFileConstant.BACK_QUOTE_STRING)) { return src.substring(1, src.length() - 1) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/PollTimer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/PollTimer.java new file mode 100644 index 0000000000000..1dd077854e8a8 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/PollTimer.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.util; + +public class PollTimer { + + private long startMs; + private long currentTimeMs; + private long deadlineMs; + private long timeoutMs; + + public PollTimer(final long startMs, final long timeoutMs) { + this.update(startMs); + this.reset(timeoutMs); + } + + public boolean isExpired() { + return this.currentTimeMs >= this.deadlineMs; + } + + public boolean isExpired(final long deltaMs) { + return this.currentTimeMs >= this.deadlineMs - Math.max(deltaMs, 0); + } + + public boolean notExpired() { + return !this.isExpired(); + } + + public boolean notExpired(final long deltaMs) { + return !this.isExpired(deltaMs); + } + + public void reset(final long timeoutMs) { + if (timeoutMs < 0L) { + throw new IllegalArgumentException("Invalid negative timeout " + timeoutMs); + } else { + this.timeoutMs = timeoutMs; + this.startMs = this.currentTimeMs; + if (this.currentTimeMs > Long.MAX_VALUE - timeoutMs) { + this.deadlineMs = Long.MAX_VALUE; + } else { + this.deadlineMs = this.currentTimeMs + timeoutMs; + } + } + } + + public void update() { + update(System.currentTimeMillis()); + } + + public void update(final long currentTimeMs) { + this.currentTimeMs = Math.max(currentTimeMs, this.currentTimeMs); + } + + public long remainingMs() { + return Math.max(0L, this.deadlineMs - this.currentTimeMs); + } + + public long currentTimeMs() { + return this.currentTimeMs; + } + + public long elapsedMs() { + return this.currentTimeMs - this.startMs; + } + + public long timeoutMs() { + return this.timeoutMs; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/SetPartitioner.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/SetPartitioner.java new file mode 100644 index 0000000000000..69e46642de4a1 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/SetPartitioner.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.util; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class SetPartitioner { + + /** + * Partitions the given set into the specified number of subsets. + * + *

Ensures that each partition contains at least one element, even if the number of elements in + * the set is less than the number of partitions. When the number of elements is greater than or + * equal to the number of partitions, elements are evenly distributed across the partitions. + * + *

Example: + * + *

    + *
  • 1 topic, 4 partitions: [topic1 | topic1 | topic1 | topic1] + *
  • 3 topics, 4 partitions: [topic1 | topic2 | topic3 | topic1] + *
  • 2 topics, 4 partitions: [topic1 | topic2 | topic1 | topic2] + *
  • 5 topics, 4 partitions: [topic1, topic4 | topic2 | topic5 | topic3] + *
  • 7 topics, 3 partitions: [topic1, topic6, topic7 | topic2, topic3 | topic5, topic4] + *
+ * + * @param set the given set + * @param partitions the number of partitions + * @param the type of the elements in the set + * @return a list containing the specified number of subsets + */ + public static List> partition(final Set set, final int partitions) { + final List> result = new ArrayList<>(partitions); + for (int i = 0; i < partitions; i++) { + result.add(new HashSet<>()); + } + + final List elements = new ArrayList<>(set); + int index = 0; + + // When the number of elements is less than the number of partitions, distribute elements + // repeatedly + for (int i = 0; i < partitions; i++) { + result.get(i).add(elements.get(index)); + index = (index + 1) % elements.size(); + } + + // When the number of elements is greater than or equal to the number of partitions, distribute + // elements normally + for (int i = partitions; i < elements.size(); i++) { + result.get(i % partitions).add(elements.get(i)); + } + + return result; + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/SubscriptionPollTimer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/SubscriptionPollTimer.java deleted file mode 100644 index 756b1fd678f42..0000000000000 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/SubscriptionPollTimer.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.util; - -public class SubscriptionPollTimer { - - private long startMs; - private long currentTimeMs; - private long deadlineMs; - private long timeoutMs; - - public SubscriptionPollTimer(long startMs, long timeoutMs) { - this.update(startMs); - this.reset(timeoutMs); - } - - public boolean isExpired() { - return this.currentTimeMs >= this.deadlineMs; - } - - public boolean notExpired() { - return !this.isExpired(); - } - - public void reset(long timeoutMs) { - if (timeoutMs < 0L) { - throw new IllegalArgumentException("Invalid negative timeout " + timeoutMs); - } else { - this.timeoutMs = timeoutMs; - this.startMs = this.currentTimeMs; - if (this.currentTimeMs > Long.MAX_VALUE - timeoutMs) { - this.deadlineMs = Long.MAX_VALUE; - } else { - this.deadlineMs = this.currentTimeMs + timeoutMs; - } - } - } - - public void update() { - update(System.currentTimeMillis()); - } - - public void update(long currentTimeMs) { - this.currentTimeMs = Math.max(currentTimeMs, this.currentTimeMs); - } - - public long remainingMs() { - return Math.max(0L, this.deadlineMs - this.currentTimeMs); - } - - public long currentTimeMs() { - return this.currentTimeMs; - } - - public long elapsedMs() { - return this.currentTimeMs - this.startMs; - } - - public long timeoutMs() { - return this.timeoutMs; - } -} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java new file mode 100644 index 0000000000000..bd767d270b530 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.util; + +public class RetryUtils { + + public interface CallableWithException { + T call() throws E; + } + + public static final int MAX_RETRIES = 3; + + public static T retryOnException( + final CallableWithException callable) throws E { + int attempt = 0; + while (true) { + try { + return callable.call(); + } catch (Exception e) { + attempt++; + if (attempt >= MAX_RETRIES) { + throw e; + } + } + } + } + + private RetryUtils() { + // utility class + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index 88ecda0524892..8534b808754e7 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -25,6 +25,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; @@ -32,7 +33,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.IMeasurementSchema; import java.nio.ByteBuffer; import java.time.LocalDate; @@ -44,11 +45,12 @@ public class SessionUtils { private static final byte TYPE_NULL = -2; + private static final int EMPTY_DATE_INT = 10000101; public static ByteBuffer getTimeBuffer(Tablet tablet) { - ByteBuffer timeBuffer = ByteBuffer.allocate(tablet.getTimeBytesSize()); - for (int i = 0; i < tablet.rowSize; i++) { - timeBuffer.putLong(tablet.timestamps[i]); + ByteBuffer timeBuffer = ByteBuffer.allocate(getTimeBytesSize(tablet)); + for (int i = 0; i < tablet.getRowSize(); i++) { + timeBuffer.putLong(tablet.getTimestamp(i)); } timeBuffer.flip(); return timeBuffer; @@ -56,20 +58,18 @@ public static ByteBuffer getTimeBuffer(Tablet tablet) { @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public static ByteBuffer getValueBuffer(Tablet tablet) { - ByteBuffer valueBuffer = ByteBuffer.allocate(tablet.getTotalValueOccupation()); + ByteBuffer valueBuffer = ByteBuffer.allocate(getTotalValueOccupation(tablet)); for (int i = 0; i < tablet.getSchemas().size(); i++) { - MeasurementSchema schema = tablet.getSchemas().get(i); + IMeasurementSchema schema = tablet.getSchemas().get(i); getValueBufferOfDataType(schema.getType(), tablet, i, valueBuffer); } - if (tablet.bitMaps != null) { - for (BitMap bitMap : tablet.bitMaps) { - boolean columnHasNull = bitMap != null && !bitMap.isAllUnmarked(); + BitMap[] bitMaps = tablet.getBitMaps(); + if (bitMaps != null) { + for (BitMap bitMap : bitMaps) { + boolean columnHasNull = bitMap != null && !bitMap.isAllUnmarked(tablet.getRowSize()); valueBuffer.put(BytesUtils.boolToByte(columnHasNull)); if (columnHasNull) { - byte[] bytes = bitMap.getByteArray(); - for (int j = 0; j < tablet.rowSize / Byte.SIZE + 1; j++) { - valueBuffer.put(bytes[j]); - } + valueBuffer.put(bitMap.getTruncatedByteArray(tablet.getRowSize())); } } } @@ -77,6 +77,74 @@ public static ByteBuffer getValueBuffer(Tablet tablet) { return valueBuffer; } + private static int getTimeBytesSize(Tablet tablet) { + return tablet.getRowSize() * 8; + } + + /** + * @return Total bytes of values + */ + private static int getTotalValueOccupation(Tablet tablet) { + int valueOccupation = 0; + int columnIndex = 0; + List schemas = tablet.getSchemas(); + int rowSize = tablet.getRowSize(); + for (IMeasurementSchema schema : schemas) { + valueOccupation += + calOccupationOfOneColumn(schema.getType(), tablet.getValues(), columnIndex, rowSize); + columnIndex++; + } + + // Add bitmap size if the tablet has bitMaps + BitMap[] bitMaps = tablet.getBitMaps(); + if (bitMaps != null) { + for (BitMap bitMap : bitMaps) { + // Marker byte + valueOccupation++; + if (bitMap != null && !bitMap.isAllUnmarked()) { + valueOccupation += rowSize / Byte.SIZE + 1; + } + } + } + return valueOccupation; + } + + private static int calOccupationOfOneColumn( + TSDataType dataType, Object[] values, int columnIndex, int rowSize) { + int valueOccupation = 0; + switch (dataType) { + case BOOLEAN: + valueOccupation += rowSize; + break; + case INT32: + case FLOAT: + case DATE: + valueOccupation += rowSize * 4; + break; + case INT64: + case DOUBLE: + case TIMESTAMP: + valueOccupation += rowSize * 8; + break; + case TEXT: + case BLOB: + case STRING: + valueOccupation += rowSize * 4; + Binary[] binaries = (Binary[]) values[columnIndex]; + for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { + valueOccupation += + binaries[rowIndex] != null + ? binaries[rowIndex].getLength() + : Binary.EMPTY_VALUE.getLength(); + } + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", dataType)); + } + return valueOccupation; + } + public static ByteBuffer getValueBuffer(List types, List values) throws IoTDBConnectionException { ByteBuffer buffer = ByteBuffer.allocate(SessionUtils.calculateLength(types, values)); @@ -198,11 +266,9 @@ private static void getValueBufferOfDataType( switch (dataType) { case INT32: - int[] intValues = (int[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - if (tablet.bitMaps == null - || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].isMarked(index)) { + int[] intValues = (int[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i)) { valueBuffer.putInt(intValues[index]); } else { valueBuffer.putInt(Integer.MIN_VALUE); @@ -211,11 +277,9 @@ private static void getValueBufferOfDataType( break; case INT64: case TIMESTAMP: - long[] longValues = (long[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - if (tablet.bitMaps == null - || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].isMarked(index)) { + long[] longValues = (long[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i)) { valueBuffer.putLong(longValues[index]); } else { valueBuffer.putLong(Long.MIN_VALUE); @@ -223,11 +287,9 @@ private static void getValueBufferOfDataType( } break; case FLOAT: - float[] floatValues = (float[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - if (tablet.bitMaps == null - || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].isMarked(index)) { + float[] floatValues = (float[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i)) { valueBuffer.putFloat(floatValues[index]); } else { valueBuffer.putFloat(Float.MIN_VALUE); @@ -235,11 +297,9 @@ private static void getValueBufferOfDataType( } break; case DOUBLE: - double[] doubleValues = (double[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - if (tablet.bitMaps == null - || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].isMarked(index)) { + double[] doubleValues = (double[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i)) { valueBuffer.putDouble(doubleValues[index]); } else { valueBuffer.putDouble(Double.MIN_VALUE); @@ -247,11 +307,9 @@ private static void getValueBufferOfDataType( } break; case BOOLEAN: - boolean[] boolValues = (boolean[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - if (tablet.bitMaps == null - || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].isMarked(index)) { + boolean[] boolValues = (boolean[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i)) { valueBuffer.put(BytesUtils.boolToByte(boolValues[index])); } else { valueBuffer.put(BytesUtils.boolToByte(false)); @@ -261,16 +319,25 @@ private static void getValueBufferOfDataType( case TEXT: case STRING: case BLOB: - Binary[] binaryValues = (Binary[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - valueBuffer.putInt(binaryValues[index].getLength()); - valueBuffer.put(binaryValues[index].getValues()); + Binary[] binaryValues = (Binary[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i) && binaryValues[index] != null) { + valueBuffer.putInt(binaryValues[index].getLength()); + valueBuffer.put(binaryValues[index].getValues()); + } else { + valueBuffer.putInt(Binary.EMPTY_VALUE.getLength()); + valueBuffer.put(Binary.EMPTY_VALUE.getValues()); + } } break; case DATE: - LocalDate[] dateValues = (LocalDate[]) tablet.values[i]; - for (int index = 0; index < tablet.rowSize; index++) { - valueBuffer.putInt(DateUtils.parseDateExpressionToInt(dateValues[index])); + LocalDate[] dateValues = (LocalDate[]) tablet.getValues()[i]; + for (int index = 0; index < tablet.getRowSize(); index++) { + if (!tablet.isNull(index, i) && dateValues[index] != null) { + valueBuffer.putInt(DateUtils.parseDateExpressionToInt(dateValues[index])); + } else { + valueBuffer.putInt(EMPTY_DATE_INT); + } } break; default: @@ -279,6 +346,20 @@ private static void getValueBufferOfDataType( } } + /* Used for table model insert only. */ + public static boolean isTabletContainsSingleDevice(Tablet tablet) { + if (tablet.getRowSize() == 1) { + return true; + } + IDeviceID firstDeviceId = tablet.getDeviceID(0); + for (int i = 1; i < tablet.getRowSize(); ++i) { + if (!firstDeviceId.equals(tablet.getDeviceID(i))) { + return false; + } + } + return true; + } + public static List parseSeedNodeUrls(List nodeUrls) { if (nodeUrls == null) { throw new NumberFormatException("nodeUrls is null"); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java index 69a45806079fa..40268a54cd340 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java @@ -32,8 +32,12 @@ import org.apache.iotdb.service.rpc.thrift.TSInsertTabletReq; import org.apache.iotdb.service.rpc.thrift.TSInsertTabletsReq; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; @@ -55,7 +59,7 @@ public class SessionCacheLeaderTest { new ArrayList() { { add(new TEndPoint("127.0.0.1", 55560)); // default endpoint - add(new TEndPoint("127.0.0.1", 55561)); // meta leader endpoint + add(new TEndPoint("127.0.0.1", 55561)); add(new TEndPoint("127.0.0.1", 55562)); add(new TEndPoint("127.0.0.1", 55563)); } @@ -78,6 +82,20 @@ public static TEndPoint getDeviceIdBelongedEndpoint(String deviceId) { return endpoints.get(deviceId.hashCode() % endpoints.size()); } + public static TEndPoint getDeviceIdBelongedEndpoint(IDeviceID deviceId) { + if (deviceId.equals(new StringArrayDeviceID("table1", "id0"))) { + return endpoints.get(0); + } else if (deviceId.equals(new StringArrayDeviceID("table1", "id1"))) { + return endpoints.get(1); + } else if (deviceId.equals(new StringArrayDeviceID("table1", "id2"))) { + return endpoints.get(2); + } else if (deviceId.equals(new StringArrayDeviceID("table1", "id3"))) { + return endpoints.get(3); + } + + return endpoints.get(deviceId.hashCode() % endpoints.size()); + } + @Test public void testInsertRecord() throws IoTDBConnectionException, StatementExecutionException { // without leader cache @@ -437,28 +455,28 @@ public void testInsertTablet() throws IoTDBConnectionException, StatementExecuti assertNull(session.endPointToSessionConnection); String deviceId = "root.sg2.d2"; - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); Tablet tablet = new Tablet(deviceId, schemaList, 100); long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = new Random().nextLong(); - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -474,21 +492,21 @@ public void testInsertTablet() throws IoTDBConnectionException, StatementExecuti assertEquals(1, session.endPointToSessionConnection.size()); for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; + int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { long value = new Random().nextLong(); - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); } - if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { session.insertTablet(tablet, true); tablet.reset(); } timestamp++; } - if (tablet.rowSize != 0) { + if (tablet.getRowSize() != 0) { session.insertTablet(tablet); tablet.reset(); } @@ -515,7 +533,7 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut add("root.sg4.d1"); } }; - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); @@ -531,19 +549,19 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 3; i++) { long value = new Random().nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { session.insertTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -552,7 +570,7 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut timestamp++; } - if (tablet1.rowSize != 0) { + if (tablet1.getRowSize() != 0) { session.insertTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -570,19 +588,19 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut assertEquals(1, session.endPointToSessionConnection.size()); for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 3; i++) { long value = new Random().nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { session.insertTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -591,7 +609,7 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut timestamp++; } - if (tablet1.rowSize != 0) { + if (tablet1.getRowSize() != 0) { session.insertTablets(tabletMap, true); tablet1.reset(); tablet2.reset(); @@ -606,6 +624,87 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut session.close(); } + @Test + public void testInsertRelationalTablet() + throws IoTDBConnectionException, StatementExecutionException { + // without leader cache + session = new MockSession("127.0.0.1", 55560, false, "table"); + session.open(); + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + + String tableName = "table1"; + List measurements = new ArrayList<>(); + measurements.add("id"); + measurements.add("s1"); + measurements.add("s2"); + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.STRING); + dataTypes.add(TSDataType.INT64); + dataTypes.add(TSDataType.INT64); + List columnTypeList = new ArrayList<>(); + columnTypeList.add(ColumnCategory.TAG); + columnTypeList.add(ColumnCategory.FIELD); + columnTypeList.add(ColumnCategory.FIELD); + Tablet tablet = new Tablet(tableName, measurements, dataTypes, columnTypeList, 50); + long timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(measurements.get(0), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(measurements.get(s), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + timestamp++; + } + + if (tablet.getRowSize() != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + session.close(); + + // with leader cache + session = new MockSession("127.0.0.1", 55560, true, "table"); + session.open(); + assertEquals(0, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(1, session.endPointToSessionConnection.size()); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(measurements.get(0), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(measurements.get(s), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + timestamp++; + } + + if (tablet.getRowSize() != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + assertEquals(4, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(4, session.endPointToSessionConnection.size()); + session.close(); + } + @Test public void testInsertRecordsWithSessionBroken() throws StatementExecutionException { // without leader cache @@ -657,6 +756,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept if (time != 0 && time % 100 == 0) { try { session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + Assert.fail(); } catch (IoTDBConnectionException e) { Assert.assertEquals( "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", @@ -671,6 +771,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept try { session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + fail(); } catch (IoTDBConnectionException e) { Assert.assertEquals( "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", e.getMessage()); @@ -685,8 +786,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept try { session.close(); } catch (IoTDBConnectionException e) { - Assert.assertEquals( - "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", e.getMessage()); + Assert.fail(e.getMessage()); } // with leader cache @@ -736,8 +836,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept try { session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); } catch (IoTDBConnectionException e) { - Assert.assertEquals( - "the session connection = TEndPoint(ip:127.0.0.1, port:55561) is broken", e.getMessage()); + Assert.fail(e.getMessage()); } assertEquals(3, session.deviceIdToEndpoint.size()); for (Map.Entry endPointMap : session.deviceIdToEndpoint.entrySet()) { @@ -758,7 +857,7 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept try { session.open(); } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } assertNull(session.deviceIdToEndpoint); assertNull(session.endPointToSessionConnection); @@ -774,7 +873,7 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept add("root.sg4.d1"); } }; - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); @@ -790,21 +889,22 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept long timestamp = System.currentTimeMillis(); for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 3; i++) { long value = new Random().nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { try { session.insertTablets(tabletMap, true); + fail(); } catch (IoTDBConnectionException e) { assertEquals( "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", @@ -817,24 +917,12 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept timestamp++; } - if (tablet1.rowSize != 0) { - try { - session.insertTablets(tabletMap, true); - } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); - } - - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - assertNull(session.deviceIdToEndpoint); assertNull(session.endPointToSessionConnection); try { session.close(); } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } // with leader cache @@ -843,29 +931,29 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept try { session.open(); } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } assertEquals(0, session.deviceIdToEndpoint.size()); assertEquals(1, session.endPointToSessionConnection.size()); for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 3; i++) { long value = new Random().nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { try { session.insertTablets(tabletMap, true); } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } tablet1.reset(); tablet2.reset(); @@ -879,28 +967,28 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept // set connection as broken, due to we enable the cache leader, when we called // ((MockSession) session).getLastConstructedSessionConnection(), the session's endpoint has // been changed to EndPoint(ip:127.0.0.1, port:55562) - Assert.assertEquals( + assertEquals( "MockSessionConnection{ endPoint=TEndPoint(ip:127.0.0.1, port:55562)}", ((MockSession) session).getLastConstructedSessionConnection().toString()); for (long row = 0; row < 10; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; + int row1 = tablet1.getRowSize(); + int row2 = tablet2.getRowSize(); + int row3 = tablet3.getRowSize(); tablet1.addTimestamp(row1, timestamp); tablet2.addTimestamp(row2, timestamp); tablet3.addTimestamp(row3, timestamp); for (int i = 0; i < 3; i++) { long value = new Random().nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + tablet1.addValue(schemaList.get(i).getMeasurementName(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementName(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementName(), row3, value); } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) { try { session.insertTablets(tabletMap, true); } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } tablet1.reset(); tablet2.reset(); @@ -911,8 +999,7 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept try { session.insertTablets(tabletMap, true); } catch (IoTDBConnectionException e) { - Assert.assertEquals( - "the session connection = TEndPoint(ip:127.0.0.1, port:55562) is broken", e.getMessage()); + fail(e.getMessage()); } tablet1.reset(); tablet2.reset(); @@ -923,6 +1010,142 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept assertEquals(getDeviceIdBelongedEndpoint(endPointEntry.getKey()), endPointEntry.getValue()); } assertEquals(3, session.endPointToSessionConnection.size()); + try { + session.close(); + } catch (IoTDBConnectionException e) { + fail(e.getMessage()); + } + } + + @Test + public void testInsertRelationalTabletWithSessionBroken() throws StatementExecutionException { + // without leader cache + session = new MockSession("127.0.0.1", 55560, false, "table"); + try { + session.open(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + + // set the session connection as broken + ((MockSession) session).getLastConstructedSessionConnection().setConnectionBroken(true); + + String tableName = "table1"; + List schemaList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + columnTypeList.add(ColumnCategory.TAG); + columnTypeList.add(ColumnCategory.FIELD); + columnTypeList.add(ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypeList, + 50); + long timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insertRelationalTablet(tablet); + fail(); + } catch (IoTDBConnectionException e) { + assertEquals( + "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", + e.getMessage()); + } + tablet.reset(); + } + timestamp++; + } + + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + try { + session.close(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + + // with leader cache + // rest the session connection + session = new MockSession("127.0.0.1", 55560, true, "table"); + try { + session.open(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + assertEquals(0, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(1, session.endPointToSessionConnection.size()); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insertRelationalTablet(tablet); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + tablet.reset(); + } + timestamp++; + } + + // set the session connection as broken + ((MockSession) session).getLastConstructedSessionConnection().setConnectionBroken(true); + // set connection as broken, due to we enable the cache leader, when we called + // ((MockSession) session).getLastConstructedSessionConnection(), the session's endpoint has + // been changed to EndPoint(ip:127.0.0.1, port:55562) + Assert.assertEquals( + "MockSessionConnection{ endPoint=TEndPoint(ip:127.0.0.1, port:55562)}", + ((MockSession) session).getLastConstructedSessionConnection().toString()); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementName(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + try { + session.insertRelationalTablet(tablet); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + tablet.reset(); + } + timestamp++; + } + + assertEquals(3, session.tableModelDeviceIdToEndpoint.size()); + for (Map.Entry endPointEntry : + session.tableModelDeviceIdToEndpoint.entrySet()) { + assertEquals(getDeviceIdBelongedEndpoint(endPointEntry.getKey()), endPointEntry.getValue()); + } + assertEquals(3, session.endPointToSessionConnection.size()); try { session.close(); } catch (IoTDBConnectionException e) { @@ -976,6 +1199,12 @@ public MockSession(String host, int rpcPort, boolean enableRedirection) { this.enableAutoFetch = false; } + public MockSession(String host, int rpcPort, boolean enableRedirection, String sqlDialect) { + this(host, rpcPort, enableRedirection); + this.sqlDialect = sqlDialect; + this.enableAutoFetch = false; + } + @Override public SessionConnection constructSessionConnection( Session session, TEndPoint endpoint, ZoneId zoneId) { @@ -995,7 +1224,7 @@ static class MockSessionConnection extends SessionConnection { private IoTDBConnectionException ioTDBConnectionException; public MockSessionConnection(Session session, TEndPoint endPoint, ZoneId zoneId) { - super(); + super(session.sqlDialect); this.endPoint = endPoint; ioTDBConnectionException = new IoTDBConnectionException( @@ -1056,7 +1285,18 @@ protected void insertTablet(TSInsertTabletReq request) if (isConnectionBroken()) { throw ioTDBConnectionException; } - throw new RedirectException(getDeviceIdBelongedEndpoint(request.prefixPath)); + if (request.writeToTable) { + if (request.size >= 50) { + // multi devices + List endPoints = new ArrayList<>(); + for (int i = 0; i < request.size; i++) { + endPoints.add(endpoints.get(i % 4)); + } + throw new RedirectException(endPoints); + } + } else { + throw new RedirectException(getDeviceIdBelongedEndpoint(request.prefixPath)); + } } @Override diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java index 61a5abd386566..a10d9b1d6f4da 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java @@ -84,7 +84,7 @@ public class SessionConnectionTest { @Before public void setUp() throws IoTDBConnectionException, StatementExecutionException, TException { MockitoAnnotations.initMocks(this); - sessionConnection = new SessionConnection(); + sessionConnection = new SessionConnection("tree"); Whitebox.setInternalState(sessionConnection, "transport", transport); Whitebox.setInternalState(sessionConnection, "client", client); session = @@ -172,7 +172,9 @@ public void testBuildSessionConnection() throws IoTDBConnectionException { ZoneId.systemDefault(), () -> Collections.singletonList(new TEndPoint("local", 12)), SessionConfig.MAX_RETRY_COUNT, - SessionConfig.RETRY_INTERVAL_IN_MS); + SessionConfig.RETRY_INTERVAL_IN_MS, + "tree", + null); } @Test(expected = IoTDBConnectionException.class) @@ -192,7 +194,9 @@ public void testBuildSessionConnection2() throws IoTDBConnectionException { ZoneId.systemDefault(), () -> Collections.singletonList(new TEndPoint("local", 12)), SessionConfig.MAX_RETRY_COUNT, - SessionConfig.RETRY_INTERVAL_IN_MS); + SessionConfig.RETRY_INTERVAL_IN_MS, + "tree", + null); } @Test diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java index 4ecd3f436bbb9..1ea2fcf6f8621 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java @@ -35,6 +35,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -799,12 +800,12 @@ public void testSetSchemaTemplateException() @Test public void testInsertTablet() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -818,12 +819,12 @@ public void testInsertTablet() throws IoTDBConnectionException, StatementExecuti @Test public void testInsertTabletOutOfOrder() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {5l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -837,12 +838,12 @@ public void testInsertTabletOutOfOrder() @Test public void testInsertAlignedTablet() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.INT32); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.INT32); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; int[][] values = new int[][] {{12, 22}, {14, 34}}; @@ -856,12 +857,12 @@ public void testInsertAlignedTablet() @Test public void testInsertAlignedTablet2() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.INT32); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.INT32); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; int[][] values = new int[][] {{12, 22}, {14, 34}}; @@ -875,12 +876,12 @@ public void testInsertAlignedTablet2() @Test public void testInsertTabletsSorted() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.INT32); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.INT32); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {2l, 1l}; int[][] values = new int[][] {{34, 42}, {40, 42}}; @@ -901,12 +902,12 @@ public void testInsertTabletsSorted() @Test public void testInsertAlignedTablets() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.FLOAT); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.FLOAT); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {2l, 1l}; float[][] values = new float[][] {{1.1f, 1.0f}, {1.2f, 1.0f}}; @@ -920,12 +921,12 @@ public void testInsertAlignedTablets() @Test public void testInsertAlignedTabletsSorted() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.DOUBLE); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.DOUBLE); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {2l, 1l}; double[][] values = new double[][] {{22.2, 22.0}, {21.5, 23.0}}; @@ -940,12 +941,12 @@ public void testInsertAlignedTabletsSorted() public void testInsertAlignedTabletsSortedEnableRedirection() throws IoTDBConnectionException, StatementExecutionException { Whitebox.setInternalState(session, "enableRedirection", false); - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.TEXT); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.TEXT); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {2l, 3l}; Binary[][] values = @@ -972,12 +973,12 @@ public void testInsertAlignedTabletsSortedEnableRedirection() @Test public void testTestInsertTablet() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -991,12 +992,12 @@ public void testTestInsertTablet() throws IoTDBConnectionException, StatementExe @Test public void testTestInsertTabletSorted() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -1009,12 +1010,12 @@ public void testTestInsertTabletSorted() @Test public void testTestInsertTablets() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -1028,12 +1029,12 @@ public void testTestInsertTablets() throws IoTDBConnectionException, StatementEx @Test public void testTestInsertTabletsSorted() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/TabletTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/TabletTest.java index 689ebab468466..bc9469e57a4a4 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/TabletTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/TabletTest.java @@ -21,10 +21,14 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; import java.util.ArrayList; import java.util.List; @@ -34,12 +38,20 @@ public class TabletTest { @Test public void testSortTablet() { Session session = new Session("127.0.0.1", 1234); - List schemaList = new ArrayList<>(); + List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s2", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s4", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s5", TSDataType.BOOLEAN)); + schemaList.add(new MeasurementSchema("s6", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s7", TSDataType.BLOB)); + schemaList.add(new MeasurementSchema("s8", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("s9", TSDataType.STRING)); + ; // insert three rows data Tablet tablet = new Tablet("root.sg1.d1", schemaList, 3); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; + Object[] values = tablet.getValues(); /* inorder data before inserting @@ -49,15 +61,35 @@ public void testSortTablet() { 1 2 */ // inorder timestamps - timestamps[0] = 2; - timestamps[1] = 0; - timestamps[2] = 1; - // just one column INT64 data - long[] sensor = (long[]) values[0]; - sensor[0] = 0; - sensor[1] = 1; - sensor[2] = 2; - tablet.rowSize = 3; + tablet.setTimestamps(new long[] {2, 0, 1}); + + values[0] = new long[] {0, 1, 2}; + values[1] = new long[] {0, 1, 2}; + values[2] = new int[] {0, 1, 2}; + values[3] = + new LocalDate[] {LocalDate.ofEpochDay(0), LocalDate.ofEpochDay(1), LocalDate.ofEpochDay(2)}; + values[4] = new boolean[] {true, false, true}; + values[5] = new double[] {0.0, 1.0, 2.0}; + values[6] = + new Binary[] { + new Binary("0".getBytes(StandardCharsets.UTF_8)), + new Binary("1".getBytes(StandardCharsets.UTF_8)), + new Binary("2".getBytes(StandardCharsets.UTF_8)) + }; + values[7] = + new Binary[] { + new Binary("0".getBytes(StandardCharsets.UTF_8)), + new Binary("1".getBytes(StandardCharsets.UTF_8)), + new Binary("2".getBytes(StandardCharsets.UTF_8)) + }; + values[8] = + new Binary[] { + new Binary("0".getBytes(StandardCharsets.UTF_8)), + new Binary("1".getBytes(StandardCharsets.UTF_8)), + new Binary("2".getBytes(StandardCharsets.UTF_8)) + }; + + tablet.setRowSize(3); session.sortTablet(tablet); @@ -71,11 +103,37 @@ public void testSortTablet() { If the data equal to above tablet, test pass, otherwise test fialed */ - long[] resTimestamps = tablet.timestamps; - long[] resValues = (long[]) tablet.values[0]; + long[] resTimestamps = tablet.getTimestamps(); long[] expectedTimestamps = new long[] {0, 1, 2}; - long[] expectedValues = new long[] {1, 2, 0}; assertArrayEquals(expectedTimestamps, resTimestamps); - assertArrayEquals(expectedValues, resValues); + assertArrayEquals(new long[] {1, 2, 0}, ((long[]) tablet.getValues()[0])); + assertArrayEquals(new long[] {1, 2, 0}, ((long[]) tablet.getValues()[1])); + assertArrayEquals(new int[] {1, 2, 0}, ((int[]) tablet.getValues()[2])); + assertArrayEquals( + new LocalDate[] {LocalDate.ofEpochDay(1), LocalDate.ofEpochDay(2), LocalDate.ofEpochDay(0)}, + ((LocalDate[]) tablet.getValues()[3])); + assertArrayEquals(new boolean[] {false, true, true}, ((boolean[]) tablet.getValues()[4])); + assertArrayEquals(new double[] {1.0, 2.0, 0.0}, ((double[]) tablet.getValues()[5]), 0.001); + assertArrayEquals( + new Binary[] { + new Binary("1".getBytes(StandardCharsets.UTF_8)), + new Binary("2".getBytes(StandardCharsets.UTF_8)), + new Binary("0".getBytes(StandardCharsets.UTF_8)) + }, + ((Binary[]) tablet.getValues()[6])); + assertArrayEquals( + new Binary[] { + new Binary("1".getBytes(StandardCharsets.UTF_8)), + new Binary("2".getBytes(StandardCharsets.UTF_8)), + new Binary("0".getBytes(StandardCharsets.UTF_8)) + }, + ((Binary[]) tablet.getValues()[7])); + assertArrayEquals( + new Binary[] { + new Binary("1".getBytes(StandardCharsets.UTF_8)), + new Binary("2".getBytes(StandardCharsets.UTF_8)), + new Binary("0".getBytes(StandardCharsets.UTF_8)) + }, + ((Binary[]) tablet.getValues()[8])); } } diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolExceptionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolExceptionTest.java index cfe1a3f5ce7f7..285fd8270dd6f 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolExceptionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolExceptionTest.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -117,12 +118,12 @@ public void testInsertTablet() throws IoTDBConnectionException, StatementExecuti Mockito.doThrow(new IoTDBConnectionException("")) .when(session) .insertTablet(any(Tablet.class), anyBoolean()); - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -136,12 +137,12 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut Mockito.doThrow(new IoTDBConnectionException("")) .when(session) .insertTablets(anyMap(), anyBoolean()); - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; Object[] values = new Object[] {true, false}; @@ -178,12 +179,12 @@ public void testInsertAlignedTablets() Mockito.doThrow(new IoTDBConnectionException("")) .when(session) .insertAlignedTablets(anyMap(), anyBoolean()); - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; Object[] values = new Object[] {true, false}; diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolTest.java index 1cf3f760c6b02..650d0087eb603 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolTest.java @@ -50,6 +50,7 @@ import org.apache.tsfile.read.common.block.column.TsBlockSerde; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -212,12 +213,12 @@ public void tearDown() { @Test public void testInsertTablet() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -256,12 +257,12 @@ public void testTimeZone() throws IoTDBConnectionException, StatementExecutionEx @Test public void testTestInsertTablet1() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -275,12 +276,12 @@ public void testTestInsertTablet1() throws IoTDBConnectionException, StatementEx @Test public void testTestInsertTablet2() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -294,12 +295,12 @@ public void testTestInsertTablet2() throws IoTDBConnectionException, StatementEx @Test public void testTestInsertTablets() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -316,12 +317,12 @@ public void testTestInsertTablets() throws IoTDBConnectionException, StatementEx @Test public void testTestInsertTablets2() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; boolean[][] values = new boolean[][] {{true, false}, {true, false}}; @@ -395,12 +396,12 @@ public void testTestInsertRecord2() throws IoTDBConnectionException, StatementEx @Test public void testInsertAlignedTablet() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; Object[] values = new Object[] {true, false}; @@ -414,12 +415,12 @@ public void testInsertAlignedTablet() @Test public void testInsertTablets() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; Object[] values = new Object[] {true, false}; @@ -436,12 +437,12 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut @Test public void testInsertAlignedTablets() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; Object[] values = new Object[] {true, false}; @@ -1188,12 +1189,12 @@ public void testShowPathsTemplateUsingOn() @Test public void testSortTablet() throws IoTDBConnectionException, StatementExecutionException { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); long[] timestamp = new long[] {1l, 2l}; Object[] values = new Object[] {true, false}; @@ -1261,7 +1262,10 @@ public void testExecuteQueryStatement2() 10, true, 10, - ZoneId.systemDefault()); + ZoneId.systemDefault(), + 1000, + false, + execResp.getColumnIndex2TsBlockColumnIndexList()); Mockito.when(session.executeQueryStatement(any(String.class), eq(50))) .thenReturn(sessionDataSet); sessionDataSetWrapper = sessionPool.executeQueryStatement(sql, 50); @@ -1472,6 +1476,8 @@ public void testExecuteQueryStatement() dataTypeList.add("INT32"); dataTypeList.add("FLOAT"); + List columnIndex2TsBlockColumnIndexList = Arrays.asList(0, 1, 2, 3); + Mockito.when(execResp.isSetColumns()).thenReturn(true); Mockito.when(execResp.getColumns()).thenReturn(columns); Mockito.when(execResp.isSetDataTypeList()).thenReturn(true); @@ -1480,6 +1486,9 @@ public void testExecuteQueryStatement() Mockito.when(execResp.getOperationType()).thenReturn("QUERY"); Mockito.when(execResp.isSetQueryId()).thenReturn(true); Mockito.when(execResp.getQueryId()).thenReturn(queryId); + Mockito.when(execResp.isIgnoreTimeStamp()).thenReturn(false); + Mockito.when(execResp.getColumnIndex2TsBlockColumnIndexList()) + .thenReturn(columnIndex2TsBlockColumnIndexList); SessionDataSet sessionDataSet = new SessionDataSet( @@ -1496,7 +1505,10 @@ public void testExecuteQueryStatement() 10, true, 10, - ZoneId.systemDefault()); + ZoneId.systemDefault(), + 1000, + false, + execResp.getColumnIndex2TsBlockColumnIndexList()); Mockito.when(session.executeQueryStatement(any(String.class))).thenReturn(sessionDataSet); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index 7fb93e49a2342..5818a89625e06 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -28,27 +28,30 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class SessionUtilsTest { @Test public void testGetTimeBuffer() { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); - long[] timestamp = new long[] {1l, 2l}; + long[] timestamp = new long[] {1L, 2L}; Object[] values = new Object[] {true, false}; BitMap[] partBitMap = new BitMap[2]; Tablet tablet = new Tablet("device1", schemas, timestamp, values, partBitMap, 2); @@ -58,48 +61,48 @@ public void testGetTimeBuffer() { @Test public void testGetValueBuffer() { - List schemas = new ArrayList<>(); + List schemas = new ArrayList<>(); MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.INT32); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.INT32); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.INT64); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.INT64); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.FLOAT); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.FLOAT); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.DOUBLE); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.DOUBLE); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.TEXT); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.TEXT); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); schema = new MeasurementSchema(); - schema.setMeasurementId("pressure"); - schema.setType(TSDataType.BOOLEAN); - schema.setCompressor(CompressionType.SNAPPY.serialize()); - schema.setEncoding(TSEncoding.PLAIN.serialize()); + schema.setMeasurementName("pressure"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); schemas.add(schema); - long[] timestamp = new long[] {1l}; + long[] timestamp = new long[] {1L}; Object[] values = new Object[6]; values[0] = new int[] {1, 2}; - values[1] = new long[] {1l, 2l}; + values[1] = new long[] {1L, 2L}; values[2] = new float[] {1.1f, 1.2f}; values[3] = new double[] {0.707, 0.708}; values[4] = @@ -113,7 +116,7 @@ public void testGetValueBuffer() { @Test public void testGetValueBuffer2() throws IoTDBConnectionException { - List valueList = Arrays.asList(12, 13l, 1.2f, 0.707, "test", false); + List valueList = Arrays.asList(12, 13L, 1.2f, 0.707, "test", false); List typeList = Arrays.asList( TSDataType.INT32, @@ -127,41 +130,102 @@ public void testGetValueBuffer2() throws IoTDBConnectionException { valueList = new ArrayList<>(); valueList.add(null); - typeList = Arrays.asList(TSDataType.INT32); + typeList = Collections.singletonList(TSDataType.INT32); timeBuffer = SessionUtils.getValueBuffer(typeList, valueList); Assert.assertNotNull(timeBuffer); - valueList = Arrays.asList(false); - typeList = Arrays.asList(TSDataType.UNKNOWN); + valueList = Collections.singletonList(false); + typeList = Collections.singletonList(TSDataType.UNKNOWN); try { - timeBuffer = SessionUtils.getValueBuffer(typeList, valueList); + SessionUtils.getValueBuffer(typeList, valueList); } catch (Exception e) { Assert.assertTrue(e instanceof IoTDBConnectionException); } } + @Test + public void testGetValueBuffer3() { + List schemas = new ArrayList<>(); + MeasurementSchema schema = new MeasurementSchema(); + schema.setMeasurementName("pressure0"); + schema.setDataType(TSDataType.INT32); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + schema = new MeasurementSchema(); + schema.setMeasurementName("pressure1"); + schema.setDataType(TSDataType.INT64); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + schema = new MeasurementSchema(); + schema.setMeasurementName("pressure2"); + schema.setDataType(TSDataType.FLOAT); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + schema = new MeasurementSchema(); + schema.setMeasurementName("pressure3"); + schema.setDataType(TSDataType.DOUBLE); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + schema = new MeasurementSchema(); + schema.setMeasurementName("pressure4"); + schema.setDataType(TSDataType.TEXT); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + schema = new MeasurementSchema(); + schema.setMeasurementName("pressure5"); + schema.setDataType(TSDataType.BOOLEAN); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + schema = new MeasurementSchema(); + schema.setMeasurementName("pressure6"); + schema.setDataType(TSDataType.DATE); + schema.setCompressionType(CompressionType.SNAPPY); + schema.setEncoding(TSEncoding.PLAIN); + schemas.add(schema); + + Tablet tablet = new Tablet("device1", schemas, 2); + tablet.setTimestamps(new long[] {1L}); + tablet.getValues()[0] = new int[] {1, 2}; + tablet.getValues()[1] = new long[] {1L, 2L}; + tablet.getValues()[2] = new float[] {1.1f, 1.2f}; + tablet.getValues()[3] = new double[] {0.707, 0.708}; + tablet.getValues()[4] = new Binary[] {null, new Binary(new byte[] {(byte) 16})}; + tablet.getValues()[5] = new boolean[] {true, false}; + tablet.getValues()[6] = new LocalDate[] {null, LocalDate.of(2024, 4, 1)}; + tablet.setRowSize(tablet.getRowSize() + 2); + + ByteBuffer timeBuffer = SessionUtils.getValueBuffer(tablet); + Assert.assertNotNull(timeBuffer); + } + @Test public void testParseSeedNodeUrls() { - List nodeUrls = Arrays.asList("127.0.0.1:1234"); + List nodeUrls = Collections.singletonList("127.0.0.1:1234"); List tEndPoints = SessionUtils.parseSeedNodeUrls(nodeUrls); Assert.assertEquals(tEndPoints.size(), 1); try { - tEndPoints = SessionUtils.parseSeedNodeUrls(null); + SessionUtils.parseSeedNodeUrls(null); } catch (Exception e) { Assert.assertTrue(e instanceof NumberFormatException); } - nodeUrls = Arrays.asList("127.0.0.1:1234:0"); + nodeUrls = Collections.singletonList("127.0.0.1:1234:0"); try { - tEndPoints = SessionUtils.parseSeedNodeUrls(nodeUrls); + SessionUtils.parseSeedNodeUrls(nodeUrls); } catch (Exception e) { Assert.assertTrue(e instanceof NumberFormatException); } - nodeUrls = Arrays.asList("127.0.0.1:test"); + nodeUrls = Collections.singletonList("127.0.0.1:test"); try { - tEndPoints = SessionUtils.parseSeedNodeUrls(nodeUrls); + SessionUtils.parseSeedNodeUrls(nodeUrls); } catch (Exception e) { Assert.assertTrue(e instanceof NumberFormatException); } @@ -169,7 +233,7 @@ public void testParseSeedNodeUrls() { @Test public void testParseSeedNodeUrlsException() { - List nodeUrls = Arrays.asList("127.0.0.1:1234"); + List nodeUrls = Collections.singletonList("127.0.0.1:1234"); List tEndPoints = SessionUtils.parseSeedNodeUrls(nodeUrls); Assert.assertEquals(tEndPoints.size(), 1); } diff --git a/iotdb-core/ainode/.gitignore b/iotdb-core/ainode/.gitignore new file mode 100644 index 0000000000000..ddf402ed3b39f --- /dev/null +++ b/iotdb-core/ainode/.gitignore @@ -0,0 +1,11 @@ +# generated by Thrift +/ainode/thrift/ + +# generated by maven +/ainode/conf/ + +# .whl of ainode, generated by Poetry +/dist/ + +# the config to build ainode, it will be generated automatically +pyproject.toml diff --git a/iotdb-core/ainode/README.md b/iotdb-core/ainode/README.md new file mode 100644 index 0000000000000..150ad93e499b0 --- /dev/null +++ b/iotdb-core/ainode/README.md @@ -0,0 +1,22 @@ + + +# Apache IoTDB AINode \ No newline at end of file diff --git a/iotdb-core/ainode/README_ZH.md b/iotdb-core/ainode/README_ZH.md new file mode 100644 index 0000000000000..150ad93e499b0 --- /dev/null +++ b/iotdb-core/ainode/README_ZH.md @@ -0,0 +1,22 @@ + + +# Apache IoTDB AINode \ No newline at end of file diff --git a/iotdb-core/ainode/ainode.xml b/iotdb-core/ainode/ainode.xml new file mode 100644 index 0000000000000..f11314c7c5aa0 --- /dev/null +++ b/iotdb-core/ainode/ainode.xml @@ -0,0 +1,89 @@ + + + + ainode-assembly + + dir + zip + + + + README.md + + + README_ZH.md + + + ${maven.multiModuleProjectDirectory}/LICENSE-binary + LICENSE + + + ${maven.multiModuleProjectDirectory}/NOTICE-binary + NOTICE + + + + + resources/conf + conf + + + resources/sbin + sbin + 0755 + + + dist + lib + + *.whl + + + + ${project.basedir}/../../scripts/conf + conf + + ainode-env.* + **/ainode-env.* + + 0755 + + + ${project.basedir}/../../scripts/sbin + sbin + + *ainode.* + **/*ainode.* + + 0755 + + + ${project.basedir}/../../scripts/tools + tools + + *ainode.* + **/*ainode.* + + 0755 + + + diff --git a/iotdb-core/ainode/ainode/TimerXL/__init__.py b/iotdb-core/ainode/ainode/TimerXL/__init__.py new file mode 100644 index 0000000000000..4b8ee97fad2be --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/TimerXL/layers/Attn_Bias.py b/iotdb-core/ainode/ainode/TimerXL/layers/Attn_Bias.py new file mode 100644 index 0000000000000..d374039579ac0 --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/layers/Attn_Bias.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import abc +import math +import torch +from einops import rearrange +from torch import nn + + +class AttentionBias(nn.Module, abc.ABC): + def __init__(self, dim: int, num_heads: int): + super().__init__() + assert num_heads > 0 and dim % num_heads == 0 + + self.num_heads = num_heads + self.head_dim = dim // num_heads + + @abc.abstractmethod + def forward(self, query_id, kv_id): ... + + +class BinaryAttentionBias(AttentionBias): + def __init__(self, dim: int, num_heads: int): + super().__init__(dim, num_heads) + self.emb = nn.Embedding(num_embeddings=2, embedding_dim=self.num_heads) + + def forward(self, query_id, kv_id): + ind = torch.eq(query_id.unsqueeze(-1), kv_id.unsqueeze(-2)) + weight = rearrange( + self.emb.weight, "two num_heads -> two num_heads 1 1") + bias = ~ind * weight[:1] + ind * weight[1:] + return bias + + +def _relative_position_bucket(relative_position, bidirectional=True, num_buckets=32, max_distance=128): + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets += (relative_position > + 0).to(torch.long) * num_buckets + relative_position = torch.abs(relative_position) + else: + relative_position = - \ + torch.min(relative_position, torch.zeros_like(relative_position)) + + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + relative_position_if_large = max_exact + ( + torch.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(torch.long) + relative_position_if_large = torch.min( + relative_position_if_large, torch.full_like( + relative_position_if_large, num_buckets - 1) + ) + + relative_buckets += torch.where(is_small, + relative_position, relative_position_if_large) + return relative_buckets + + +class T5AttentionBias(AttentionBias): + def __init__(self, dim: int, num_heads: int): + super().__init__(dim, num_heads) + self.num_buckets = 32 + self.max_distance = 32 + self.relative_attention_bias = nn.Embedding(self.num_buckets, 1) + + def forward(self, n_vars, n_tokens): + context_position = torch.arange(n_tokens, dtype=torch.long,)[:, None] + memory_position = torch.arange(n_tokens, dtype=torch.long, )[None, :] + relative_position = memory_position - context_position + bucket = _relative_position_bucket(relative_position=relative_position, bidirectional=False, + num_buckets=self.num_buckets, max_distance=self.max_distance).to(self.relative_attention_bias.weight.device) + bias = self.relative_attention_bias(bucket).squeeze(-1) + bias = bias.reshape(1, 1, bias.shape[0], bias.shape[1]) + mask1 = torch.ones((n_vars, n_vars), dtype=torch.bool).to(bias.device) + final_bias = torch.kron(mask1, bias) + return final_bias diff --git a/iotdb-core/ainode/ainode/TimerXL/layers/Attn_Projection.py b/iotdb-core/ainode/ainode/TimerXL/layers/Attn_Projection.py new file mode 100644 index 0000000000000..f53c849b441a1 --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/layers/Attn_Projection.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import abc +import torch +from functools import cached_property +from einops import einsum, rearrange, repeat +from torch import nn + + +class Projection(nn.Module, abc.ABC): + def __init__(self, proj_width: int, num_heads: int, **kwargs): + super().__init__() + self.proj_width = proj_width + self.num_heads = num_heads + + @abc.abstractmethod + def forward(self, x, seq_id): ... + + +class RotaryProjection(Projection): + def __init__(self, *, proj_width: int, num_heads: int, max_len: int = 512, base: int = 10000): + super().__init__(proj_width, num_heads) + assert ( + self.proj_width % 2 == 0 + ), f"proj_width must be even, got {self.proj_width}" + self.register_buffer( + "theta", + 1.0 + / torch.pow( + base, + torch.arange(0, self.proj_width, 2, dtype=torch.float) + / self.proj_width, + ), + persistent=False, + ) + self.register_buffer("cos", None, persistent=False) + self.register_buffer("sin", None, persistent=False) + self._init_freq(max_len=max_len) + + def _init_freq(self, max_len: int): + if self.cos is None or self.cos.size(-2) < max_len: + position = torch.arange( + max_len, device=self.theta.device, dtype=self.theta.dtype + ) + m_theta = einsum(position, self.theta, + "length, width -> length width") + m_theta = repeat(m_theta, "length width -> length (width 2)") + self.register_buffer("cos", torch.cos(m_theta), persistent=False) + self.register_buffer("sin", torch.sin(m_theta), persistent=False) + + @staticmethod + def _rotate(x): + x1, x2 = rearrange(x, "... (dim r) -> r ... dim", r=2) + return rearrange([-x2, x1], "r ... dim -> ... (dim r)", r=2) # noqa + + def forward(self, x, seq_id): + self._init_freq(max_len=seq_id.max() + 1) + rot_cos = self.cos[seq_id] + rot_sin = self.sin[seq_id] + return rot_cos * x + rot_sin * self._rotate(x) + + +class QueryKeyProjection(nn.Module): + def __init__(self, dim: int, num_heads: int, proj_layer, kwargs=None, partial_factor=None): + super().__init__() + if partial_factor is not None: + assert ( + 0.0 <= partial_factor[0] < partial_factor[1] <= 1.0 + ), f"got {partial_factor[0]}, {partial_factor[1]}" + assert num_heads > 0 and dim % num_heads == 0 + + self.head_dim = dim // num_heads + self.partial_factor = partial_factor + self.query_proj = proj_layer( + proj_width=self.proj_width, + num_heads=num_heads, + **(kwargs or {}), + ) + self.key_proj = self.query_proj + + @cached_property + def proj_width(self) -> int: + if self.partial_factor is None: + return self.head_dim + return int(self.head_dim * (self.partial_factor[1] - self.partial_factor[0])) + + @cached_property + def split_sizes(self): + if self.partial_factor is None: + return 0, self.head_dim, 0 + return ( + int(self.partial_factor[0] * self.head_dim), + self.proj_width, + int((1.0 - self.partial_factor[1]) * self.head_dim), + ) + + def forward(self, query, key, query_id, kv_id): + if self.partial_factor is not None: + queries = list(query.split(self.split_sizes, dim=-1)) + keys = list(key.split(self.split_sizes, dim=-1)) + queries[1] = self.query_proj(queries[1], seq_id=query_id) + keys[1] = self.key_proj(keys[1], seq_id=kv_id) + query = torch.cat(queries, dim=-1) + key = torch.cat(keys, dim=-1) + else: + query = self.query_proj(query, seq_id=query_id) + key = self.key_proj(key, seq_id=kv_id) + return query, key diff --git a/iotdb-core/ainode/ainode/TimerXL/layers/Embed.py b/iotdb-core/ainode/ainode/TimerXL/layers/Embed.py new file mode 100644 index 0000000000000..33747b2fe60ba --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/layers/Embed.py @@ -0,0 +1,261 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import math +import torch +import torch.nn as nn +from torch.jit import is_scripting +from ainode.TimerXL.models.configuration_timer import TimerxlConfig + +class PositionalEmbedding(nn.Module): + def __init__(self, d_model, max_len=6500): + super(PositionalEmbedding, self).__init__() + # Compute the positional encodings once in log space. + pe = torch.zeros(max_len, d_model).float() + pe.require_grad = False + + position = torch.arange(0, max_len).float().unsqueeze(1) + div_term = (torch.arange(0, d_model, 2).float() + * -(math.log(10000.0) / d_model)).exp() + + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + + def forward(self, x): + return self.pe[:, :x.size(1)] + + +class TokenEmbedding(nn.Module): + def __init__(self, c_in, d_model): + super(TokenEmbedding, self).__init__() + padding = 1 if torch.__version__ >= '1.5.0' else 2 + self.tokenConv = nn.Conv1d(in_channels=c_in, out_channels=d_model, + kernel_size=3, padding=padding, padding_mode='circular', bias=False) + for m in self.modules(): + if isinstance(m, nn.Conv1d): + nn.init.kaiming_normal_( + m.weight, mode='fan_in', nonlinearity='leaky_relu') + + def forward(self, x): + x = self.tokenConv(x.permute(0, 2, 1)).transpose(1, 2) + return x + + +class FixedEmbedding(nn.Module): + def __init__(self, c_in, d_model): + super(FixedEmbedding, self).__init__() + + w = torch.zeros(c_in, d_model).float() + w.require_grad = False + + position = torch.arange(0, c_in).float().unsqueeze(1) + div_term = (torch.arange(0, d_model, 2).float() + * -(math.log(10000.0) / d_model)).exp() + + w[:, 0::2] = torch.sin(position * div_term) + w[:, 1::2] = torch.cos(position * div_term) + + self.emb = nn.Embedding(c_in, d_model) + self.emb.weight = nn.Parameter(w, requires_grad=False) + + def forward(self, x): + return self.emb(x).detach() + + +class TemporalEmbedding(nn.Module): + def __init__(self, d_model, embed_type='fixed', freq='h'): + super(TemporalEmbedding, self).__init__() + + minute_size = 4 + hour_size = 24 + weekday_size = 7 + day_size = 32 + month_size = 13 + + Embed = FixedEmbedding if embed_type == 'fixed' else nn.Embedding + if freq == 't': + self.minute_embed = Embed(minute_size, d_model) + self.hour_embed = Embed(hour_size, d_model) + self.weekday_embed = Embed(weekday_size, d_model) + self.day_embed = Embed(day_size, d_model) + self.month_embed = Embed(month_size, d_model) + + def forward(self, x): + x = x.long() + minute_x = self.minute_embed(x[:, :, 4]) if hasattr( + self, 'minute_embed') else 0. + hour_x = self.hour_embed(x[:, :, 3]) + weekday_x = self.weekday_embed(x[:, :, 2]) + day_x = self.day_embed(x[:, :, 1]) + month_x = self.month_embed(x[:, :, 0]) + + return hour_x + weekday_x + day_x + month_x + minute_x + + +class TimeFeatureEmbedding(nn.Module): + def __init__(self, d_model, embed_type='timeF', freq='h'): + super(TimeFeatureEmbedding, self).__init__() + + freq_map = {'h': 4, 't': 5, 's': 6, + 'm': 1, 'a': 1, 'w': 2, 'd': 3, 'b': 3} + d_inp = freq_map[freq] + self.embed = nn.Linear(d_inp, d_model, bias=False) + + def forward(self, x): + return self.embed(x) + + +class DataEmbedding(nn.Module): + def __init__(self, c_in, d_model, embed_type='fixed', freq='h', dropout=0.1): + super(DataEmbedding, self).__init__() + + self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model) + self.position_embedding = PositionalEmbedding(d_model=d_model) + self.temporal_embedding = TemporalEmbedding(d_model=d_model, embed_type=embed_type, + freq=freq) if embed_type != 'timeF' else TimeFeatureEmbedding( + d_model=d_model, embed_type=embed_type, freq=freq) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, x_mark): + if x_mark is None: + x = self.value_embedding(x) + self.position_embedding(x) + else: + x = self.value_embedding( + x) + self.temporal_embedding(x_mark) + self.position_embedding(x) + return self.dropout(x) + + +class DataEmbedding_inverted(nn.Module): + def __init__(self, c_in, d_model, embed_type='fixed', freq='h', dropout=0.1): + super(DataEmbedding_inverted, self).__init__() + self.value_embedding = nn.Linear(c_in, d_model) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, x_mark): + x = x.permute(0, 2, 1) + # x: [Batch Variate Time] + if x_mark is None: + x = self.value_embedding(x) + else: + x = self.value_embedding( + torch.cat([x, x_mark.permute(0, 2, 1)], 1)) + # x: [Batch Variate d_model] + return self.dropout(x) + + +class DataEmbedding_wo_pos(nn.Module): + def __init__(self, c_in, d_model, embed_type='fixed', freq='h', dropout=0.1): + super(DataEmbedding_wo_pos, self).__init__() + + self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model) + self.position_embedding = PositionalEmbedding(d_model=d_model) + self.temporal_embedding = TemporalEmbedding(d_model=d_model, embed_type=embed_type, + freq=freq) if embed_type != 'timeF' else TimeFeatureEmbedding( + d_model=d_model, embed_type=embed_type, freq=freq) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, x_mark): + if x_mark is None: + x = self.value_embedding(x) + else: + x = self.value_embedding(x) + self.temporal_embedding(x_mark) + return self.dropout(x) + + +class PatchEmbedding(nn.Module): + def __init__(self, d_model, patch_len, stride, padding, dropout): + super(PatchEmbedding, self).__init__() + # Patching + self.patch_len = patch_len + self.stride = stride + self.padding_patch_layer = nn.ReplicationPad1d((0, padding)) + + # Backbone, Input encoding: projection of feature vectors onto a d-dim vector space + self.value_embedding = nn.Linear(patch_len, d_model, bias=False) + + # Positional embedding + self.position_embedding = PositionalEmbedding(d_model) + + # Residual dropout + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + # do patching + n_vars = x.shape[1] + x = self.padding_patch_layer(x) + x = x.unfold(dimension=-1, size=self.patch_len, step=self.stride) + x = torch.reshape(x, (x.shape[0] * x.shape[1], x.shape[2], x.shape[3])) + # Input encoding + x = self.value_embedding(x) + self.position_embedding(x) + return self.dropout(x), n_vars + +class TimerPatchEmbedding(nn.Module): + def __init__(self, config: TimerxlConfig): + super().__init__() + self.input_token_len = config.input_token_len + self.emb = nn.Linear(config.input_token_len, + config.hidden_size, bias=False) + + def forward(self, hidden_state: torch.Tensor): + hidden_state = hidden_state.unfold( + dimension=-1, size=self.input_token_len, step=self.input_token_len) + return self.emb(hidden_state) + +class TimeMoeRotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings=10000, base=10000, device=None): + super().__init__() + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.max_seq_len_cached: int = 0 + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, + 2, dtype=torch.int64).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len: int, device: torch.device, dtype: torch.dtype): + self.max_seq_len_cached = int(seq_len) + t = torch.arange(self.max_seq_len_cached, device=device, + dtype=torch.int64).type_as(self.inv_freq) + + freqs = torch.outer(t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + if not is_scripting(): + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + else: + self.cos_cached = emb.cos().to(dtype) + self.sin_cached = emb.sin().to(dtype) + + def forward(self, x, seq_len: int=0): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache( + seq_len=seq_len, device=x.device, dtype=x.dtype) + + return ( + self.cos_cached[:seq_len].to(dtype=x.dtype), + self.sin_cached[:seq_len].to(dtype=x.dtype), + ) diff --git a/iotdb-core/ainode/ainode/TimerXL/layers/SelfAttention_Family.py b/iotdb-core/ainode/ainode/TimerXL/layers/SelfAttention_Family.py new file mode 100644 index 0000000000000..82ac54fffce3e --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/layers/SelfAttention_Family.py @@ -0,0 +1,167 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import torch +import torch.nn as nn +import torch.nn.functional as F +from typing import Optional, Tuple, Any +import numpy as np +from math import sqrt +from einops import repeat +from ainode.TimerXL.layers.Attn_Bias import BinaryAttentionBias +from ainode.TimerXL.layers.Attn_Projection import QueryKeyProjection, RotaryProjection +from ainode.TimerXL.layers.Embed import TimeMoeRotaryEmbedding +from ainode.core.util.masking import TriangularCausalMask, TimerMultivariateMask, TimerCovariateMask +from ainode.TimerXL.models.configuration_timer import TimerxlConfig +from ainode.core.util.huggingface_cache import Cache, DynamicCache + +def rotate_half(x): + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2:] + return torch.cat((-x2, x1), dim=-1) + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + +class FullAttention(nn.Module): + def __init__(self, mask_flag=True, scale=None, attention_dropout=0.1, output_attention=False): + super(FullAttention, self).__init__() + self.scale = scale + self.mask_flag = mask_flag + self.output_attention = output_attention + self.dropout = nn.Dropout(attention_dropout) + + def forward(self, queries, keys, values, attn_mask, n_vars=None, n_tokens=None, tau=None, delta=None): + B, L, H, E = queries.shape + _, S, _, D = values.shape + scale = self.scale or 1. / sqrt(E) + + scores = torch.einsum("blhe,bshe->bhls", queries, keys) + + if self.mask_flag: + if attn_mask is None: + attn_mask = TriangularCausalMask(B, L, device=queries.device) + + scores.masked_fill_(attn_mask.mask, -np.inf) + + A = self.dropout(torch.softmax(scale * scores, dim=-1)) + V = torch.einsum("bhls,bshd->blhd", A, values) + + if self.output_attention: + return V.contiguous(), A + else: + return V.contiguous(), None + + +class TimerAttention(nn.Module): + def __init__(self, config: TimerxlConfig, layer_idx: Optional[int] = None): + super().__init__() + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.attention_dropout = config.attention_dropout + self.q_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) + self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + self.rotary_emb = TimeMoeRotaryEmbedding( + self.head_dim, max_position_embeddings=config.max_position_embeddings) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional["Cache"] = None, + ) -> Tuple[torch.Tensor, Optional["Cache"]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view( + bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view( + bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view( + bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length( + kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb( + query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + key_states, value_states = past_key_value.update( + key_states, value_states, self.layer_idx) + + attn_output = F.scaled_dot_product_attention( + query_states, key_states, value_states, attention_mask, dropout_p=self.attention_dropout) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + attn_output = self.o_proj(attn_output) + + return attn_output, past_key_value + + +class AttentionLayer(nn.Module): + def __init__(self, attention, d_model, n_heads, d_keys=None, d_values=None): + super(AttentionLayer, self).__init__() + + d_keys = d_keys or (d_model // n_heads) + d_values = d_values or (d_model // n_heads) + + self.inner_attention = attention + self.query_projection = nn.Linear(d_model, d_keys * n_heads) + self.key_projection = nn.Linear(d_model, d_keys * n_heads) + self.value_projection = nn.Linear(d_model, d_values * n_heads) + self.out_projection = nn.Linear(d_values * n_heads, d_model) + self.n_heads = n_heads + + def forward(self, queries, keys, values, attn_mask, n_vars=None, n_tokens=None, tau=None, delta=None): + B, L, _ = queries.shape + _, S, _ = keys.shape + H = self.n_heads + + queries = self.query_projection(queries).view(B, L, H, -1) + keys = self.key_projection(keys).view(B, S, H, -1) + values = self.value_projection(values).view(B, S, H, -1) + + out, attn = self.inner_attention( + queries, + keys, + values, + attn_mask, + n_vars=n_vars, + n_tokens=n_tokens, + tau=tau, + delta=delta + ) + out = out.view(B, L, -1) + + return self.out_projection(out), attn + diff --git a/iotdb-core/ainode/ainode/TimerXL/layers/Transformer_EncDec.py b/iotdb-core/ainode/ainode/TimerXL/layers/Transformer_EncDec.py new file mode 100644 index 0000000000000..ceb3d7ebe1653 --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/layers/Transformer_EncDec.py @@ -0,0 +1,328 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import torch +import torch.nn as nn +import torch.nn.functional as F +from typing import Optional, Tuple +from ainode.TimerXL.models.configuration_timer import TimerxlConfig +from ainode.TimerXL.layers.SelfAttention_Family import TimerAttention +from ainode.core.util.activation import ACT2FN +from ainode.core.util.huggingface_cache import Cache, DynamicCache + +class EncoderLayer(nn.Module): + def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"): + super(EncoderLayer, self).__init__() + d_ff = d_ff or 4 * d_model + self.attention = attention + self.conv1 = nn.Conv1d(in_channels=d_model, + out_channels=d_ff, kernel_size=1) + self.conv2 = nn.Conv1d( + in_channels=d_ff, out_channels=d_model, kernel_size=1) + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + self.activation = F.relu if activation == "relu" else F.gelu + + def forward(self, x, attn_mask=None, tau=None, delta=None): + new_x, attn = self.attention( + x, x, x, + attn_mask=attn_mask, + tau=tau, delta=delta + ) + x = x + self.dropout(new_x) + + y = x = self.norm1(x) + y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1)))) + y = self.dropout(self.conv2(y).transpose(-1, 1)) + + return self.norm2(x + y), attn + + +class DecoderLayer(nn.Module): + def __init__(self, self_attention, cross_attention, d_model, d_ff=None, + dropout=0.1, activation="relu"): + super(DecoderLayer, self).__init__() + d_ff = d_ff or 4 * d_model + self.self_attention = self_attention + self.cross_attention = cross_attention + self.conv1 = nn.Conv1d(in_channels=d_model, + out_channels=d_ff, kernel_size=1) + self.conv2 = nn.Conv1d( + in_channels=d_ff, out_channels=d_model, kernel_size=1) + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + self.activation = F.relu if activation == "relu" else F.gelu + + def forward(self, x, cross, x_mask=None, cross_mask=None, tau=None, delta=None): + x = x + self.dropout(self.self_attention( + x, x, x, + attn_mask=x_mask, + tau=tau, delta=None + )[0]) + x = self.norm1(x) + + x = x + self.dropout(self.cross_attention( + x, cross, cross, + attn_mask=cross_mask, + tau=tau, delta=delta + )[0]) + + y = x = self.norm2(x) + y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1)))) + y = self.dropout(self.conv2(y).transpose(-1, 1)) + + return self.norm3(x + y) + + +class DecoderOnlyLayer(nn.Module): + def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"): + super(DecoderOnlyLayer, self).__init__() + d_ff = d_ff or 4 * d_model + self.attention = attention + self.conv1 = nn.Conv1d(in_channels=d_model, + out_channels=d_ff, kernel_size=1) + self.conv2 = nn.Conv1d( + in_channels=d_ff, out_channels=d_model, kernel_size=1) + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + self.activation = F.relu if activation == "relu" else F.gelu + + def forward(self, x, attn_mask=None, tau=None, delta=None): + new_x, attn = self.attention( + x, x, x, + attn_mask=attn_mask, + tau=tau, delta=delta + ) + x = x + self.dropout(new_x) + + y = x = self.norm1(x) + y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1)))) + y = self.dropout(self.conv2(y).transpose(-1, 1)) + + return self.norm2(x + y), attn + + +class TimerLayer(nn.Module): + def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"): + super(TimerLayer, self).__init__() + d_ff = d_ff or 4 * d_model + self.attention = attention + self.conv1 = nn.Conv1d(in_channels=d_model, + out_channels=d_ff, kernel_size=1) + self.conv2 = nn.Conv1d( + in_channels=d_ff, out_channels=d_model, kernel_size=1) + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + self.activation = F.relu if activation == "relu" else F.gelu + + def forward(self, x, n_vars, n_tokens, attn_mask=None, tau=None, delta=None): + new_x, attn = self.attention( + x, x, x, + n_vars=n_vars, + n_tokens=n_tokens, + attn_mask=attn_mask, + tau=tau, delta=delta + ) + x = x + self.dropout(new_x) + + y = x = self.norm1(x) + y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1)))) + y = self.dropout(self.conv2(y).transpose(-1, 1)) + + return self.norm2(x + y), attn + + +class Encoder(nn.Module): + def __init__(self, attn_layers, conv_layers=None, norm_layer=None): + super(Encoder, self).__init__() + self.attn_layers = nn.ModuleList(attn_layers) + self.conv_layers = nn.ModuleList( + conv_layers) if conv_layers is not None else None + self.norm = norm_layer + + def forward(self, x, attn_mask=None, tau=None, delta=None): + # x [B, L, D] + attns = [] + if self.conv_layers is not None: + for i, (attn_layer, conv_layer) in enumerate(zip(self.attn_layers, self.conv_layers)): + delta = delta if i == 0 else None + x, attn = attn_layer( + x, attn_mask=attn_mask, tau=tau, delta=delta) + x = conv_layer(x) + attns.append(attn) + x, attn = self.attn_layers[-1](x, tau=tau, delta=None) + attns.append(attn) + else: + for attn_layer in self.attn_layers: + x, attn = attn_layer( + x, attn_mask=attn_mask, tau=tau, delta=delta) + attns.append(attn) + + if self.norm is not None: + x = self.norm(x) + + return x, attns + + +class Decoder(nn.Module): + def __init__(self, layers, norm_layer=None, projection=None): + super(Decoder, self).__init__() + self.layers = nn.ModuleList(layers) + self.norm = norm_layer + self.projection = projection + + def forward(self, x, cross, x_mask=None, cross_mask=None, tau=None, delta=None): + for layer in self.layers: + x = layer(x, cross, x_mask=x_mask, + cross_mask=cross_mask, tau=tau, delta=delta) + + if self.norm is not None: + x = self.norm(x) + + if self.projection is not None: + x = self.projection(x) + return x + + +class DecoderOnly(nn.Module): + def __init__(self, attn_layers, conv_layers=None, norm_layer=None): + super(DecoderOnly, self).__init__() + self.attn_layers = nn.ModuleList(attn_layers) + self.conv_layers = nn.ModuleList( + conv_layers) if conv_layers is not None else None + self.norm = norm_layer + + def forward(self, x, attn_mask=None, tau=None, delta=None): + # x [B, L, D] + attns = [] + if self.conv_layers is not None: + for i, (attn_layer, conv_layer) in enumerate(zip(self.attn_layers, self.conv_layers)): + delta = delta if i == 0 else None + x, attn = attn_layer( + x, attn_mask=attn_mask, tau=tau, delta=delta) + x = conv_layer(x) + attns.append(attn) + x, attn = self.attn_layers[-1](x, tau=tau, delta=None) + attns.append(attn) + else: + for attn_layer in self.attn_layers: + x, attn = attn_layer( + x, attn_mask=attn_mask, tau=tau, delta=delta) + attns.append(attn) + + if self.norm is not None: + x = self.norm(x) + + return x, attns + + +class TimerBlock(nn.Module): + def __init__(self, attn_layers, conv_layers=None, norm_layer=None): + super(TimerBlock, self).__init__() + self.attn_layers = nn.ModuleList(attn_layers) + self.conv_layers = nn.ModuleList( + conv_layers) if conv_layers is not None else None + self.norm = norm_layer + + def forward(self, x, n_vars, n_tokens, attn_mask=None, tau=None, delta=None): + # x [B, L, D] + attns = [] + if self.conv_layers is not None: + for i, (attn_layer, conv_layer) in enumerate(zip(self.attn_layers, self.conv_layers)): + delta = delta if i == 0 else None + x, attn = attn_layer( + x, attn_mask=attn_mask, tau=tau, delta=delta) + x = conv_layer(x) + attns.append(attn) + x, attn = self.attn_layers[-1](x, n_vars, + n_tokens, tau=tau, delta=None) + attns.append(attn) + else: + for attn_layer in self.attn_layers: + x, attn = attn_layer(x, n_vars, n_tokens, + attn_mask=attn_mask, tau=tau, delta=delta) + attns.append(attn) + + if self.norm is not None: + x = self.norm(x) + + return x, attns + +class TimerMLP(nn.Module): + def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str): + super().__init__() + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.gate_proj = nn.Linear( + self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear( + self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear( + self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[hidden_act] + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + +class TimerDecoderLayer(nn.Module): + def __init__(self, config: TimerxlConfig, layer_idx: int): + super().__init__() + self.self_attn = TimerAttention(config, layer_idx) + + self.ffn_layer = TimerMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + ) + self.norm1 = torch.nn.LayerNorm(config.hidden_size) + self.norm2 = torch.nn.LayerNorm(config.hidden_size) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + use_cache: bool = False + ) -> Tuple[torch.FloatTensor, Optional[Cache]]: + residual = hidden_states + + # Self Attention + hidden_states, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + ) + + hidden_states = residual + hidden_states + hidden_states = self.norm1(hidden_states) + + # Fully Connected + residual = hidden_states + hidden_states = self.ffn_layer(hidden_states) + hidden_states = residual + hidden_states + hidden_states = self.norm2(hidden_states) + + if not use_cache: + present_key_value = None + return hidden_states, present_key_value \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/TimerXL/layers/__init__.py b/iotdb-core/ainode/ainode/TimerXL/layers/__init__.py new file mode 100644 index 0000000000000..4b8ee97fad2be --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/layers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/TimerXL/models/__init__.py b/iotdb-core/ainode/ainode/TimerXL/models/__init__.py new file mode 100644 index 0000000000000..4b8ee97fad2be --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/models/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/TimerXL/models/configuration_timer.py b/iotdb-core/ainode/ainode/TimerXL/models/configuration_timer.py new file mode 100644 index 0000000000000..7dbce577537c5 --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/models/configuration_timer.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from typing import List + +class TimerxlConfig: + model_type = "timerxl" + + def __init__( + self, + input_token_len: int = 96, # how many points as a token, don't change + hidden_size: int = 1024, # model hidden size + intermediate_size: int = 2048, # ffn middle size + output_token_lens: List[int] = [96],# how many points as a token, don't change + num_hidden_layers: int = 8, + num_attention_heads: int = 8, + hidden_act: str = "silu", # activation function + use_cache: bool = True, # kv cache + rope_theta: int = 10000, # ROBE parameter + attention_dropout: float = 0.0, + initializer_range: float = 0.02, # be of no use, because we already have weights + max_position_embeddings: int = 10000, + ckpt_path: str = None, # weight path + **kwargs, + ): + self.input_token_len = input_token_len + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.output_token_lens = output_token_lens + self.use_cache = use_cache + self.rope_theta = rope_theta + self.attention_dropout = attention_dropout + self.initializer_range = initializer_range + self.max_position_embeddings = max_position_embeddings + self.ckpt_path = ckpt_path + + super().__init__( + **kwargs, + ) + + @classmethod + def from_dict(cls, config_dict: dict) -> "TimerxlConfig": + return cls(**config_dict) diff --git a/iotdb-core/ainode/ainode/TimerXL/models/timer_xl.py b/iotdb-core/ainode/ainode/TimerXL/models/timer_xl.py new file mode 100644 index 0000000000000..0e66542405ebf --- /dev/null +++ b/iotdb-core/ainode/ainode/TimerXL/models/timer_xl.py @@ -0,0 +1,413 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import torch +import os +from torch import nn +from typing import Optional, List, Dict, Any, Tuple +from dataclasses import dataclass +from ainode.TimerXL.layers.Transformer_EncDec import TimerDecoderLayer +from ainode.TimerXL.layers.Embed import TimerPatchEmbedding +from ainode.TimerXL.models.configuration_timer import TimerxlConfig +from ainode.core.util.masking import prepare_4d_causal_attention_mask +from ainode.core.util.huggingface_cache import Cache, DynamicCache + +from safetensors.torch import load_file as load_safetensors +from huggingface_hub import hf_hub_download + +from ainode.core.log import Logger +logger = Logger() + +@dataclass +class Output: + outputs: torch.Tensor + past_key_values: Optional[Any] = None + +class TimerModel(nn.Module): + def __init__(self, config: TimerxlConfig): + super().__init__() + self.config = config + self.embed_layer = TimerPatchEmbedding(config) + self.layers = nn.ModuleList( + [TimerDecoderLayer(config, layer_idx) + for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = torch.nn.LayerNorm(config.hidden_size) + self.gradient_checkpointing = False + + def forward( + self, + input_ids: torch.FloatTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + use_cache: bool = None, + ): + # input_ids is the input of time series, its shape is [batch_size, seq_len] + + if input_ids is not None: + batch_size, seq_length = input_ids.shape + else: + raise ValueError( + "You have to specify either decoder_input_ids or decoder_inputs_embeds") + + inputs_embeds = self.embed_layer(input_ids) + + seq_length = inputs_embeds.shape[1] + + past_key_values_length = 0 + + if use_cache: + use_legacy_cache = not isinstance(past_key_values, Cache) + if use_legacy_cache: + past_key_values = DynamicCache.from_legacy_cache( + past_key_values) + past_key_values_length = past_key_values.get_usable_length( + seq_length) + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + # 4d mask is passed through the layers + attention_mask = prepare_4d_causal_attention_mask( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + ) + + hidden_states = inputs_embeds + + # decoder layers + next_decoder_cache = None + + for decoder_layer in self.layers: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_values, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[1] + + hidden_states = self.norm(hidden_states) + + next_cache = None + if use_cache: + next_cache = next_decoder_cache.to_legacy_cache( + ) if use_legacy_cache else next_decoder_cache + + return Output( + outputs=hidden_states, + past_key_values=next_cache + ) + +class TimerForPrediction(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.model = TimerModel(self.config) + lm_head_list = [] + self.output_token_len_map = {} + for i, output_token_len in enumerate(self.config.output_token_lens): + lm_head_list.append( + nn.Linear(self.config.hidden_size, output_token_len, bias=False)) + self.output_token_len_map[output_token_len] = i + self.lm_heads = nn.ModuleList(lm_head_list) + self.loss_function = torch.nn.MSELoss(reduction='none') + + def forward( + self, + input_ids: torch.FloatTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + use_cache: Optional[bool] = None, + max_output_length: Optional[int] = None, + revin: Optional[bool] = True, + ): + if revin: + means, stdev = input_ids.mean(dim=-1, keepdim=True), input_ids.std(dim=-1, keepdim=True) + input_ids = (input_ids - means) / stdev + + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + hidden_states = outputs.outputs + + if max_output_length is None: + output_token_len = self.config.output_token_lens[0] + max_output_length = output_token_len + else: + output_token_len = self.config.output_token_lens[0] + for h in self.config.output_token_lens[1:]: + if h > max_output_length: + break + else: + output_token_len = h + + lm_head = self.lm_heads[self.output_token_len_map[output_token_len]] + predictions = lm_head(hidden_states)[:, -1, :] + + if output_token_len > max_output_length: + predictions = predictions[:, :max_output_length] + if revin: + predictions = predictions * stdev + means + + return Output(predictions, outputs.past_key_values) + + +class Model(nn.Module): + """ + Timer-XL: Long-Context Transformers for Unified Time Series Forecasting + + Paper: https://arxiv.org/abs/2410.04803 + + GitHub: https://github.com/thuml/Timer-XL + + Citation: @article{liu2024timer, + title={Timer-XL: Long-Context Transformers for Unified Time Series Forecasting}, + author={Liu, Yong and Qin, Guo and Huang, Xiangdong and Wang, Jianmin and Long, Mingsheng}, + journal={arXiv preprint arXiv:2410.04803}, + year={2024} + } + """ + def __init__(self, config: TimerxlConfig): + super().__init__() + self.config = config # can't be scripted by torch + + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model = TimerForPrediction(config).to(self.device) + + if config.ckpt_path is not None and config.ckpt_path != '': + if config.ckpt_path.endswith('.pt') or config.ckpt_path.endswith('.pth'): + state_dict = torch.load(config.ckpt_path) + elif config.ckpt_path.endswith('.safetensors'): + if not os.path.exists(config.ckpt_path): + logger.info(f"Checkpoint not found at {config.ckpt_path}, downloading from HuggingFace...") + repo_id = "thuml/timer-base-84m" + try: + config.ckpt_path = hf_hub_download(repo_id=repo_id, filename=os.path.basename(config.ckpt_path), local_dir=os.path.dirname(config.ckpt_path)) + logger.info(f"Got checkpoint to {config.ckpt_path}") + except Exception as e: + logger.error(f"Failed to download checkpoint to {config.ckpt_path} due to {e}") + raise e + state_dict = load_safetensors(config.ckpt_path) + else: + raise ValueError('unsupported model weight type') + # If there is no key beginning with 'model.model' in state_dict, add a 'model.' before all keys. (The model code here has an additional layer of encapsulation compared to the code on huggingface.) + if not any(k.startswith('model.model') for k in state_dict.keys()): + state_dict = {'model.' + k: v for k, v in state_dict.items()} + self.load_state_dict(state_dict, strict=True) + + def set_device(self, device): + self.model.to(device) + self.device = next(self.model.parameters()).device + + def inference(self, x, max_new_tokens: int = 96): + # x.shape: [L, C], type: DataFrame + # here we only except C=1 temporarily + # change [L, C=1] to [batchsize=1, L] + self.device = next(self.model.parameters()).device + + x = torch.tensor(x, dtype=next(self.model.parameters()).dtype, device=self.device) + x = x.view(1, -1) + + preds = self.forward(x, max_new_tokens) + preds = preds.detach().cpu().numpy() + + return preds + + def forward(self, x, max_new_tokens: int = 96): + # self.config.is_encoder_decoder = False + self.eval() + self.device = next(self.model.parameters()).device + + if len(x.shape) == 2: + batch_size, cur_len = x.shape + if cur_len < self.config.input_token_len: + raise ValueError( + f"Input length must be at least {self.config.input_token_len}") + elif cur_len % self.config.input_token_len != 0: + new_len = (cur_len // self.config.input_token_len) * \ + self.config.input_token_len + x = x[:, -new_len:] + else: + raise ValueError('Input shape must be: [batch_size, seq_len]') + + use_cache = self.config.use_cache + all_input_ids = x + + attention_mask = self.prepare_attention_mask_for_generation(all_input_ids) + all_input_ids_length = all_input_ids.shape[-1] + max_length = max_new_tokens + all_input_ids_length + + all_input_ids = all_input_ids.to(self.device) + batch_size, cur_len = all_input_ids.shape + + unfinished_sequences = torch.ones(batch_size, dtype=torch.long, device=all_input_ids.device) + cache_position = torch.arange(cur_len, device=all_input_ids.device) + true_seq_len = cur_len // self.config.input_token_len + attention_mask = attention_mask[:, -true_seq_len:] + + this_peer_finished = False + past_key_values = None + position_ids = None + while not this_peer_finished: + ( + input_ids, + position_ids, + past_key_values, + attention_mask, + revin + ) = self.prepare_inputs_for_generation( + all_input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + # position_ids=position_ids # Wrong?! + position_ids=None # True?! based on huggingface code + ) + + input_length = all_input_ids.shape[1] + + # forward pass to get next token + outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + max_output_length=max_length - input_length, + revin=revin + ) + + next_tokens = outputs.outputs + + # update generated ids, model inputs, and length for next step + horizon_length = next_tokens.shape[1] // self.config.input_token_len + + all_input_ids = torch.cat([all_input_ids, next_tokens], dim=-1) + ( + past_key_values, + attention_mask, + cache_position + ) = self._update_model_kwargs_for_generation( + outputs, + attention_mask=attention_mask, + horizon_length=horizon_length, + cache_position=cache_position, + ) + + unfinished_sequences = unfinished_sequences & (all_input_ids.shape[1] < max_length) + this_peer_finished = unfinished_sequences.max() == 0 + + if all_input_ids.shape[1] > max_length: + all_input_ids = all_input_ids[:, :max_length] + + return all_input_ids[:, -(max_length - cur_len):] + + def prepare_attention_mask_for_generation( + self, + inputs: torch.Tensor, + ) -> torch.LongTensor: + return torch.ones(inputs.shape[:2], dtype=torch.long, device=inputs.device) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, revin=True, position_ids=None + ): + # Omit tokens covered by past_key_values + if past_key_values is not None: + if isinstance(past_key_values, Cache): + cache_length = past_key_values.get_seq_length() + if isinstance(past_key_values, DynamicCache): + past_length = past_key_values.seen_tokens + else: + past_length = cache_length + + max_cache_length = past_key_values.get_max_length() + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > (input_ids.shape[1] // self.config.input_token_len): + input_ids = input_ids[:, - + (attention_mask.shape[1] - past_length):] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < (input_ids.shape[1] // self.config.input_token_len): + input_ids = input_ids[:, past_length * + self.config.input_token_len:] + # 3 - Otherwise (past_length >= (input_ids.shape[1] // self.config.input_token_len)), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + (input_ids.shape[1] // self.config.input_token_len) > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, - + (input_ids.shape[1] // self.config.input_token_len):] + + return (input_ids, position_ids, past_key_values, attention_mask, revin) + + def _update_model_kwargs_for_generation( + self, + outputs, + attention_mask = None, + cache_position = None, + horizon_length: int = 1, + ) -> Dict[str, Any]: + # update past_key_values + past_key_values = outputs.past_key_values + + # update attention mask + if attention_mask is not None: + attention_mask = torch.cat( + [attention_mask, attention_mask.new_ones((attention_mask.shape[0], horizon_length))], dim=-1 + ) + + if cache_position is not None: + cache_position = cache_position[-1:] + horizon_length + + return (past_key_values, attention_mask, cache_position) \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/__init__.py b/iotdb-core/ainode/ainode/__init__.py new file mode 100644 index 0000000000000..2a1e720805f29 --- /dev/null +++ b/iotdb-core/ainode/ainode/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/iotdb-core/ainode/ainode/core/__init__.py b/iotdb-core/ainode/ainode/core/__init__.py new file mode 100644 index 0000000000000..2a1e720805f29 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/iotdb-core/ainode/ainode/core/client.py b/iotdb-core/ainode/ainode/core/client.py new file mode 100644 index 0000000000000..2796d46ab930d --- /dev/null +++ b/iotdb-core/ainode/ainode/core/client.py @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import time + +from thrift.Thrift import TException +from thrift.protocol import TCompactProtocol, TBinaryProtocol +from thrift.transport import TSocket, TTransport + +from ainode.core.config import AINodeDescriptor +from ainode.core.constant import TSStatusCode +from ainode.core.log import Logger +from ainode.core.util.decorator import singleton +from ainode.core.util.status import verify_success +from ainode.thrift.common.ttypes import TEndPoint, TSStatus, TAINodeLocation, TAINodeConfiguration +from ainode.thrift.confignode import IConfigNodeRPCService +from ainode.thrift.confignode.ttypes import (TAINodeRemoveReq, TNodeVersionInfo, + TAINodeRegisterReq, TAINodeRestartReq) +from ainode.thrift.confignode.ttypes import TUpdateModelInfoReq + +logger = Logger() + + +@singleton +class ClientManager(object): + def __init__(self): + self._config_node_endpoint = AINodeDescriptor().get_config().get_ain_target_config_node_list() + + def borrow_config_node_client(self): + return ConfigNodeClient(config_leader=self._config_node_endpoint) + + +class ConfigNodeClient(object): + def __init__(self, config_leader: TEndPoint): + self._config_leader = config_leader + self._config_nodes = [] + self._cursor = 0 + self._transport = None + self._client = None + + self._MSG_RECONNECTION_FAIL = "Fail to connect to any config node. Please check status of ConfigNodes" + self._RETRY_NUM = 5 + self._RETRY_INTERVAL_MS = 1 + + self._try_to_connect() + + def _try_to_connect(self) -> None: + if self._config_leader is not None: + try: + self._connect(self._config_leader) + return + except TException: + logger.warning("The current node {} may have been down, try next node", self._config_leader) + self._config_leader = None + + if self._transport is not None: + self._transport.close() + + try_host_num = 0 + while try_host_num < len(self._config_nodes): + self._cursor = (self._cursor + 1) % len(self._config_nodes) + + try_endpoint = self._config_nodes[self._cursor] + try: + self._connect(try_endpoint) + return + except TException: + logger.warning("The current node {} may have been down, try next node", try_endpoint) + + try_host_num = try_host_num + 1 + + raise TException(self._MSG_RECONNECTION_FAIL) + + def _connect(self, target_config_node: TEndPoint) -> None: + transport = TTransport.TFramedTransport( + TSocket.TSocket(target_config_node.ip, target_config_node.port) + ) + if not transport.isOpen(): + try: + transport.open() + except TTransport.TTransportException as e: + logger.error("TTransportException: {}".format(e)) + raise e + + if AINodeDescriptor().get_config().get_ain_thrift_compression_enabled(): + protocol = TCompactProtocol.TCompactProtocol(transport) + else: + protocol = TBinaryProtocol.TBinaryProtocol(transport) + self._client = IConfigNodeRPCService.Client(protocol) + + def _wait_and_reconnect(self) -> None: + # wait to start the next try + time.sleep(self._RETRY_INTERVAL_MS) + + try: + self._try_to_connect() + except TException: + # can not connect to each config node + self._sync_latest_config_node_list() + self._try_to_connect() + + def _sync_latest_config_node_list(self) -> None: + # TODO + pass + + def _update_config_node_leader(self, status: TSStatus) -> bool: + if status.code == TSStatusCode.REDIRECTION_RECOMMEND.get_status_code(): + if status.redirectNode is not None: + self._config_leader = status.redirectNode + else: + self._config_leader = None + return True + return False + + def node_register(self, cluster_name: str, configuration: TAINodeConfiguration, + version_info: TNodeVersionInfo) -> int: + req = TAINodeRegisterReq( + clusterName=cluster_name, + aiNodeConfiguration=configuration, + versionInfo=version_info + ) + + for _ in range(0, self._RETRY_NUM): + try: + resp = self._client.registerAINode(req) + if not self._update_config_node_leader(resp.status): + verify_success(resp.status, "An error occurs when calling node_register()") + self._config_nodes = resp.configNodeList + return resp.aiNodeId + except TTransport.TException: + logger.warning("Failed to connect to ConfigNode {} from AINode when executing node_register()", + self._config_leader) + self._config_leader = None + self._wait_and_reconnect() + + raise TException(self._MSG_RECONNECTION_FAIL) + + def node_restart(self, cluster_name: str, configuration: TAINodeConfiguration, + version_info: TNodeVersionInfo) -> None: + req = TAINodeRestartReq( + clusterName=cluster_name, + aiNodeConfiguration=configuration, + versionInfo=version_info + ) + + for _ in range(0, self._RETRY_NUM): + try: + resp = self._client.restartAINode(req) + if not self._update_config_node_leader(resp.status): + verify_success(resp.status, "An error occurs when calling node_restart()") + self._config_nodes = resp.configNodeList + return resp.status + except TTransport.TException: + logger.warning("Failed to connect to ConfigNode {} from AINode when executing node_restart()", + self._config_leader) + self._config_leader = None + self._wait_and_reconnect() + + raise TException(self._MSG_RECONNECTION_FAIL) + + def node_remove(self, location: TAINodeLocation): + req = TAINodeRemoveReq( + aiNodeLocation=location + ) + for _ in range(0, self._RETRY_NUM): + try: + status = self._client.removeAINode(req) + if not self._update_config_node_leader(status): + verify_success(status, "An error occurs when calling node_restart()") + return status + except TTransport.TException: + logger.warning("Failed to connect to ConfigNode {} from AINode when executing node_remove()", + self._config_leader) + self._config_leader = None + self._wait_and_reconnect() + raise TException(self._MSG_RECONNECTION_FAIL) + + def get_ainode_configuration(self, node_id: int) -> map: + for _ in range(0, self._RETRY_NUM): + try: + resp = self._client.getAINodeConfiguration(node_id) + if not self._update_config_node_leader(resp.status): + verify_success(resp.status, "An error occurs when calling get_ainode_configuration()") + return resp.aiNodeConfigurationMap + except TTransport.TException: + logger.warning("Failed to connect to ConfigNode {} from AINode when executing " + "get_ainode_configuration()", + self._config_leader) + self._config_leader = None + self._wait_and_reconnect() + raise TException(self._MSG_RECONNECTION_FAIL) + + def update_model_info(self, model_id:str, model_status:int, attribute:str = "", ainode_id=None, input_length=0, output_length=0) -> None: + if ainode_id is None: + ainode_id = [] + for _ in range(0, self._RETRY_NUM): + try: + req = TUpdateModelInfoReq( + model_id, model_status, attribute + ) + if ainode_id is not None: + req.aiNodeIds = ainode_id + req.inputLength = input_length + req.outputLength = output_length + status = self._client.updateModelInfo(req) + if not self._update_config_node_leader(status): + verify_success(status, "An error occurs when calling update model info") + return status + except TTransport.TException: + logger.warning("Failed to connect to ConfigNode {} from AINode when executing update model info", + self._config_leader) + self._config_leader = None + self._wait_and_reconnect() + raise TException(self._MSG_RECONNECTION_FAIL) diff --git a/iotdb-core/ainode/ainode/core/config.py b/iotdb-core/ainode/ainode/core/config.py new file mode 100644 index 0000000000000..036dc20b9c224 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/config.py @@ -0,0 +1,248 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os + +from ainode.core.constant import (AINODE_CONF_DIRECTORY_NAME, + AINODE_CONF_FILE_NAME, + AINODE_MODELS_DIR, AINODE_LOG_DIR, AINODE_SYSTEM_DIR, AINODE_INFERENCE_RPC_ADDRESS, + AINODE_INFERENCE_RPC_PORT, AINODE_THRIFT_COMPRESSION_ENABLED, + AINODE_SYSTEM_FILE_NAME, AINODE_CLUSTER_NAME, AINODE_VERSION_INFO, AINODE_BUILD_INFO, + AINODE_CONF_GIT_FILE_NAME, AINODE_CONF_POM_FILE_NAME, AINODE_ROOT_DIR, + AINODE_ROOT_CONF_DIRECTORY_NAME) +from ainode.core.exception import BadNodeUrlError +from ainode.core.log import Logger +from ainode.core.util.decorator import singleton +from ainode.thrift.common.ttypes import TEndPoint + +logger = Logger() + + +class AINodeConfig(object): + def __init__(self): + # Used for connection of DataNode/ConfigNode clients + self._ain_inference_rpc_address: str = AINODE_INFERENCE_RPC_ADDRESS + self._ain_inference_rpc_port: int = AINODE_INFERENCE_RPC_PORT + + # log directory + self._ain_logs_dir: str = AINODE_LOG_DIR + + # Directory to save models + self._ain_models_dir = AINODE_MODELS_DIR + + self._ain_system_dir = AINODE_SYSTEM_DIR + + # Whether to enable compression for thrift + self._ain_thrift_compression_enabled = AINODE_THRIFT_COMPRESSION_ENABLED + + # Cache number of model storage to avoid repeated loading + self._ain_model_storage_cache_size = 30 + + # Target ConfigNode to be connected by AINode + self._ain_target_config_node_list: TEndPoint = TEndPoint("127.0.0.1", 10710) + + # use for node management + self._ainode_id = 0 + self._cluster_name = AINODE_CLUSTER_NAME + + self._version_info = AINODE_VERSION_INFO + self._build_info = AINODE_BUILD_INFO + + def get_cluster_name(self) -> str: + return self._cluster_name + + def set_cluster_name(self, cluster_name: str) -> None: + self._cluster_name = cluster_name + + def get_version_info(self) -> str: + return self._version_info + + def get_ainode_id(self) -> int: + return self._ainode_id + + def set_ainode_id(self, ainode_id: int) -> None: + self._ainode_id = ainode_id + + def get_build_info(self) -> str: + return self._build_info + + def set_build_info(self, build_info: str) -> None: + self._build_info = build_info + + def set_version_info(self, version_info: str) -> None: + self._version_info = version_info + + def get_ain_inference_rpc_address(self) -> str: + return self._ain_inference_rpc_address + + def set_ain_inference_rpc_address(self, ain_inference_rpc_address: str) -> None: + self._ain_inference_rpc_address = ain_inference_rpc_address + + def get_ain_inference_rpc_port(self) -> int: + return self._ain_inference_rpc_port + + def set_ain_inference_rpc_port(self, ain_inference_rpc_port: int) -> None: + self._ain_inference_rpc_port = ain_inference_rpc_port + + def get_ain_logs_dir(self) -> str: + return self._ain_logs_dir + + def set_ain_logs_dir(self, ain_logs_dir: str) -> None: + self._ain_logs_dir = ain_logs_dir + + def get_ain_models_dir(self) -> str: + return self._ain_models_dir + + def set_ain_models_dir(self, ain_models_dir: str) -> None: + self._ain_models_dir = ain_models_dir + + def get_ain_system_dir(self) -> str: + return self._ain_system_dir + + def set_ain_system_dir(self, ain_system_dir: str) -> None: + self._ain_system_dir = ain_system_dir + + def get_ain_thrift_compression_enabled(self) -> bool: + return self._ain_thrift_compression_enabled + + def set_ain_thrift_compression_enabled(self, ain_thrift_compression_enabled: int) -> None: + self._ain_thrift_compression_enabled = ain_thrift_compression_enabled + + def get_ain_model_storage_cache_size(self) -> int: + return self._ain_model_storage_cache_size + + def get_ain_target_config_node_list(self) -> TEndPoint: + return self._ain_target_config_node_list + + def set_ain_target_config_node_list(self, ain_target_config_node_list: str) -> None: + self._ain_target_config_node_list = parse_endpoint_url(ain_target_config_node_list) + + +@singleton +class AINodeDescriptor(object): + + def __init__(self): + self._config = AINodeConfig() + self._load_config_from_file() + logger.info("AINodeDescriptor is init successfully.") + + def _load_config_from_file(self) -> None: + system_properties_file = os.path.join(self._config.get_ain_system_dir(), AINODE_SYSTEM_FILE_NAME) + if os.path.exists(system_properties_file): + system_configs = load_properties(system_properties_file) + if 'ainode_id' in system_configs: + self._config.set_ainode_id(int(system_configs['ainode_id'])) + + git_file = os.path.join(AINODE_ROOT_DIR, AINODE_ROOT_CONF_DIRECTORY_NAME, AINODE_CONF_GIT_FILE_NAME) + if os.path.exists(git_file): + git_configs = load_properties(git_file) + if 'git.commit.id.abbrev' in git_configs: + build_info = git_configs['git.commit.id.abbrev'] + if 'git.dirty' in git_configs: + if git_configs['git.dirty'] == "true": + build_info += "-dev" + self._config.set_build_info(build_info) + + pom_file = os.path.join(AINODE_ROOT_DIR, AINODE_ROOT_CONF_DIRECTORY_NAME, AINODE_CONF_POM_FILE_NAME) + if os.path.exists(pom_file): + pom_configs = load_properties(pom_file) + if 'version' in pom_configs: + self._config.set_version_info(pom_configs['version']) + + conf_file = os.path.join(AINODE_CONF_DIRECTORY_NAME, AINODE_CONF_FILE_NAME) + if not os.path.exists(conf_file): + logger.info("Cannot find AINode config file '{}', use default configuration.".format(conf_file)) + return + + # noinspection PyBroadException + try: + file_configs = load_properties(conf_file) + + config_keys = file_configs.keys() + + if 'ain_inference_rpc_address' in config_keys: + self._config.set_ain_inference_rpc_address(file_configs['ain_inference_rpc_address']) + + if 'ain_inference_rpc_port' in config_keys: + self._config.set_ain_inference_rpc_port(int(file_configs['ain_inference_rpc_port'])) + + if 'ain_models_dir' in config_keys: + self._config.set_ain_models_dir(file_configs['ain_models_dir']) + + if 'ain_system_dir' in config_keys: + self._config.set_ain_system_dir(file_configs['ain_system_dir']) + + if 'ain_seed_config_node' in config_keys: + self._config.set_ain_target_config_node_list(file_configs['ain_seed_config_node']) + + if 'cluster_name' in config_keys: + self._config.set_cluster_name(file_configs['cluster_name']) + + if 'ain_thrift_compression_enabled' in config_keys: + self._config.set_ain_thrift_compression_enabled(int(file_configs['ain_thrift_compression_enabled'])) + + if 'ain_logs_dir' in config_keys: + log_dir = file_configs['ain_logs_dir'] + self._config.set_ain_logs_dir(log_dir) + Logger(log_dir=log_dir).info(f"Successfully load config from {conf_file}.") + + except BadNodeUrlError: + logger.warning("Cannot load AINode conf file, use default configuration.") + + except Exception as e: + logger.warning("Cannot load AINode conf file caused by: {}, use default configuration. ".format(e)) + + def get_config(self) -> AINodeConfig: + return self._config + + +def load_properties(filepath, sep='=', comment_char='#'): + """ + Read the file passed as parameter as a properties file. + """ + props = {} + with open(filepath, "rt") as f: + for line in f: + l = line.strip() + if l and not l.startswith(comment_char): + key_value = l.split(sep) + key = key_value[0].strip() + value = sep.join(key_value[1:]).strip().strip('"') + props[key] = value + return props + + +def parse_endpoint_url(endpoint_url: str) -> TEndPoint: + """ Parse TEndPoint from a given endpoint url. + Args: + endpoint_url: an endpoint url, format: ip:port + Returns: + TEndPoint + Raises: + BadNodeUrlError + """ + split = endpoint_url.split(":") + if len(split) != 2: + raise BadNodeUrlError(endpoint_url) + + ip = split[0] + try: + port = int(split[1]) + result = TEndPoint(ip, port) + return result + except ValueError: + raise BadNodeUrlError(endpoint_url) diff --git a/iotdb-core/ainode/ainode/core/constant.py b/iotdb-core/ainode/ainode/core/constant.py new file mode 100644 index 0000000000000..a80ca680989c1 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/constant.py @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import inspect +import logging +import os +from enum import Enum +from typing import List + +AINODE_CONF_DIRECTORY_NAME = "conf" +AINODE_ROOT_CONF_DIRECTORY_NAME = "conf" +AINODE_CONF_FILE_NAME = "iotdb-ainode.properties" +AINODE_CONF_GIT_FILE_NAME = "git.properties" +AINODE_CONF_POM_FILE_NAME = "pom.properties" +AINODE_SYSTEM_FILE_NAME = "system.properties" +# inference_rpc_address +AINODE_INFERENCE_RPC_ADDRESS = "127.0.0.1" +AINODE_INFERENCE_RPC_PORT = 10810 +AINODE_MODELS_DIR = "data/ainode/models" +AINODE_SYSTEM_DIR = "data/ainode/system" +AINODE_LOG_DIR = "logs/ainode" +AINODE_THRIFT_COMPRESSION_ENABLED = False +# use for node management +AINODE_CLUSTER_NAME = "defaultCluster" +AINODE_VERSION_INFO = "UNKNOWN" +AINODE_BUILD_INFO = "UNKNOWN" +AINODE_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) + +# AINode log +AINODE_LOG_FILE_NAMES = ['log_ainode_all.log', + 'log_ainode_info.log', + 'log_ainode_warning.log', + 'log_ainode_error.log'] +AINODE_LOG_FILE_LEVELS = [ + logging.DEBUG, + logging.INFO, + logging.WARNING, + logging.ERROR] + +TRIAL_ID_PREFIX = "__trial_" +DEFAULT_TRIAL_ID = TRIAL_ID_PREFIX + "0" +DEFAULT_MODEL_FILE_NAME = "model.pt" +DEFAULT_CONFIG_FILE_NAME = "config.yaml" +DEFAULT_CHUNK_SIZE = 8192 + +DEFAULT_RECONNECT_TIMEOUT = 20 +DEFAULT_RECONNECT_TIMES = 3 + +STD_LEVEL = logging.INFO + + +class TSStatusCode(Enum): + SUCCESS_STATUS = 200 + REDIRECTION_RECOMMEND = 400 + AINODE_INTERNAL_ERROR = 1510 + INVALID_URI_ERROR = 1511 + INVALID_INFERENCE_CONFIG = 1512 + INFERENCE_INTERNAL_ERROR = 1520 + + def get_status_code(self) -> int: + return self.value + + +class TaskType(Enum): + FORECAST = "forecast" + + +class OptionsKey(Enum): + # common + TASK_TYPE = "task_type" + MODEL_TYPE = "model_type" + AUTO_TUNING = "auto_tuning" + INPUT_VARS = "input_vars" + + # forecast + INPUT_LENGTH = "input_length" + PREDICT_LENGTH = "predict_length" + PREDICT_INDEX_LIST = "predict_index_list" + INPUT_TYPE_LIST = "input_type_list" + + def name(self) -> str: + return self.value + + +class HyperparameterName(Enum): + # Training hyperparameter + LEARNING_RATE = "learning_rate" + EPOCHS = "epochs" + BATCH_SIZE = "batch_size" + USE_GPU = "use_gpu" + NUM_WORKERS = "num_workers" + + # Structure hyperparameter + KERNEL_SIZE = "kernel_size" + INPUT_VARS = "input_vars" + BLOCK_TYPE = "block_type" + D_MODEL = "d_model" + INNER_LAYERS = "inner_layer" + OUTER_LAYERS = "outer_layer" + + def name(self): + return self.value + + +class ForecastModelType(Enum): + DLINEAR = "dlinear" + DLINEAR_INDIVIDUAL = "dlinear_individual" + NBEATS = "nbeats" + + @classmethod + def values(cls) -> List[str]: + values = [] + for item in list(cls): + values.append(item.value) + return values + + +class ModelInputName(Enum): + DATA_X = "data_x" + TIME_STAMP_X = "time_stamp_x" + TIME_STAMP_Y = "time_stamp_y" + DEC_INP = "dec_inp" + + +class BuiltInModelType(Enum): + # forecast models + ARIMA = "_arima" + EXPONENTIAL_SMOOTHING = "_exponentialsmoothing" + NAIVE_FORECASTER = "_naiveforecaster" + STL_FORECASTER = "_stlforecaster" + + # anomaly detection models + GAUSSIAN_HMM = "_gaussianhmm" + GMM_HMM = "_gmmhmm" + STRAY = "_stray" + + # timerxl + TIMER_XL = "_timerxl" + + @classmethod + def values(cls) -> List[str]: + values = [] + for item in list(cls): + values.append(item.value) + return values + + +class AttributeName(Enum): + # forecast Attribute + PREDICT_LENGTH = "predict_length" + + # NaiveForecaster + STRATEGY = 'strategy' + SP = 'sp' + + # STLForecaster + # SP = 'sp' + SEASONAL = 'seasonal' + SEASONAL_DEG = 'seasonal_deg' + TREND_DEG = 'trend_deg' + LOW_PASS_DEG = 'low_pass_deg' + SEASONAL_JUMP = 'seasonal_jump' + TREND_JUMP = 'trend_jump' + LOSS_PASS_JUMP = 'low_pass_jump' + + # ExponentialSmoothing + DAMPED_TREND = 'damped_trend' + INITIALIZATION_METHOD = 'initialization_method' + OPTIMIZED = 'optimized' + REMOVE_BIAS = 'remove_bias' + USE_BRUTE = 'use_brute' + + # Arima + ORDER = "order" + SEASONAL_ORDER = "seasonal_order" + METHOD = "method" + MAXITER = "maxiter" + SUPPRESS_WARNINGS = "suppress_warnings" + OUT_OF_SAMPLE_SIZE = "out_of_sample_size" + SCORING = "scoring" + WITH_INTERCEPT = "with_intercept" + TIME_VARYING_REGRESSION = "time_varying_regression" + ENFORCE_STATIONARITY = "enforce_stationarity" + ENFORCE_INVERTIBILITY = "enforce_invertibility" + SIMPLE_DIFFERENCING = "simple_differencing" + MEASUREMENT_ERROR = "measurement_error" + MLE_REGRESSION = "mle_regression" + HAMILTON_REPRESENTATION = "hamilton_representation" + CONCENTRATE_SCALE = "concentrate_scale" + + # GAUSSIAN_HMM + N_COMPONENTS = "n_components" + COVARIANCE_TYPE = "covariance_type" + MIN_COVAR = "min_covar" + STARTPROB_PRIOR = "startprob_prior" + TRANSMAT_PRIOR = "transmat_prior" + MEANS_PRIOR = "means_prior" + MEANS_WEIGHT = "means_weight" + COVARS_PRIOR = "covars_prior" + COVARS_WEIGHT = "covars_weight" + ALGORITHM = "algorithm" + N_ITER = "n_iter" + TOL = "tol" + PARAMS = "params" + INIT_PARAMS = "init_params" + IMPLEMENTATION = "implementation" + + # GMMHMM + # N_COMPONENTS = "n_components" + N_MIX = "n_mix" + # MIN_COVAR = "min_covar" + # STARTPROB_PRIOR = "startprob_prior" + # TRANSMAT_PRIOR = "transmat_prior" + WEIGHTS_PRIOR = "weights_prior" + + # MEANS_PRIOR = "means_prior" + # MEANS_WEIGHT = "means_weight" + # ALGORITHM = "algorithm" + # COVARIANCE_TYPE = "covariance_type" + # N_ITER = "n_iter" + # TOL = "tol" + # INIT_PARAMS = "init_params" + # PARAMS = "params" + # IMPLEMENTATION = "implementation" + + # STRAY + ALPHA = "alpha" + K = "k" + KNN_ALGORITHM = "knn_algorithm" + P = "p" + SIZE_THRESHOLD = "size_threshold" + OUTLIER_TAIL = "outlier_tail" + + # timerxl + INPUT_TOKEN_LEN = "input_token_len" + HIDDEN_SIZE = "hidden_size" + INTERMEDIATE_SIZE = "intermediate_size" + OUTPUT_TOKEN_LENS = "output_token_lens" + NUM_HIDDEN_LAYERS = "num_hidden_layers" + NUM_ATTENTION_HEADS = "num_attention_heads" + HIDDEN_ACT = "hidden_act" + USE_CACHE = "use_cache" + ROPE_THETA = "rope_theta" + ATTENTION_DROPOUT = "attention_dropout" + INITIALIZER_RANGE = "initializer_range" + MAX_POSITION_EMBEDDINGS = "max_position_embeddings" + TIMERXL_CKPT_PATH = "ckpt_path" + + def name(self) -> str: + return self.value diff --git a/iotdb-core/ainode/ainode/core/exception.py b/iotdb-core/ainode/ainode/core/exception.py new file mode 100644 index 0000000000000..a9b8c496d65aa --- /dev/null +++ b/iotdb-core/ainode/ainode/core/exception.py @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import re + +from ainode.core.constant import DEFAULT_MODEL_FILE_NAME, DEFAULT_CONFIG_FILE_NAME + + +class _BaseError(Exception): + """Base class for exceptions in this module.""" + + def __init__(self): + self.message = None + + def __str__(self) -> str: + return self.message + + +class BadNodeUrlError(_BaseError): + def __init__(self, node_url: str): + self.message = "Bad node url: {}".format(node_url) + + +class ModelNotExistError(_BaseError): + def __init__(self, file_path: str): + self.message = "Model path is not exists: {} ".format(file_path) + + +class BadConfigValueError(_BaseError): + def __init__(self, config_name: str, config_value, hint: str = ''): + self.message = "Bad value [{0}] for config {1}. {2}".format(config_value, config_name, hint) + + +class MissingConfigError(_BaseError): + def __init__(self, config_name: str): + self.message = "Missing config: {}".format(config_name) + + +class MissingOptionError(_BaseError): + def __init__(self, config_name: str): + self.message = "Missing task option: {}".format(config_name) + + +class RedundantOptionError(_BaseError): + def __init__(self, option_name: str): + self.message = "Redundant task option: {}".format(option_name) + + +class WrongTypeConfigError(_BaseError): + def __init__(self, config_name: str, expected_type: str): + self.message = "Wrong type for config: {0}, expected: {1}".format(config_name, expected_type) + + +class UnsupportedError(_BaseError): + def __init__(self, msg: str): + self.message = "{0} is not supported in current version".format(msg) + + +class InvalidUriError(_BaseError): + def __init__(self, uri: str): + self.message = "Invalid uri: {}, there are no {} or {} under this uri.".format(uri, DEFAULT_MODEL_FILE_NAME, + DEFAULT_CONFIG_FILE_NAME) + + +class InvalidWindowArgumentError(_BaseError): + def __init__( + self, + window_interval, + window_step, + dataset_length): + self.message = f"Invalid inference input: window_interval {window_interval}, window_step {window_step}, dataset_length {dataset_length}" + + +class InferenceModelInternalError(_BaseError): + def __init__(self, msg: str): + self.message = "Inference model internal error: {0}".format(msg) + + +class BuiltInModelNotSupportError(_BaseError): + def __init__(self, msg: str): + self.message = "Built-in model not support: {0}".format(msg) + + +class WrongAttributeTypeError(_BaseError): + def __init__(self, attribute_name: str, expected_type: str): + self.message = "Wrong type for attribute: {0}, expected: {1}".format(attribute_name, expected_type) + + +class NumericalRangeException(_BaseError): + def __init__(self, attribute_name: str, value, min_value, max_value): + self.message = "Attribute {0} expect value between {1} and {2}, got {3} instead." \ + .format(attribute_name, min_value, max_value, value) + + +class StringRangeException(_BaseError): + def __init__(self, attribute_name: str, value: str, expect_value): + self.message = "Attribute {0} expect value in {1}, got {2} instead." \ + .format(attribute_name, expect_value, value) + + +class ListRangeException(_BaseError): + def __init__(self, attribute_name: str, value: list, expected_type: str): + self.message = "Attribute {0} expect value type list[{1}], got {2} instead." \ + .format(attribute_name, expected_type, value) + + +class AttributeNotSupportError(_BaseError): + def __init__(self, model_name: str, attribute_name: str): + self.message = "Attribute {0} is not supported in model {1}".format(attribute_name, model_name) + + +# This is used to extract the key message in RuntimeError instead of the traceback message +def runtime_error_extractor(error_message): + pattern = re.compile(r"RuntimeError: (.+)") + match = pattern.search(error_message) + + if match: + return match.group(1) + else: + return "" diff --git a/iotdb-core/ainode/ainode/core/handler.py b/iotdb-core/ainode/ainode/core/handler.py new file mode 100644 index 0000000000000..fc8f8f1aae74f --- /dev/null +++ b/iotdb-core/ainode/ainode/core/handler.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from ainode.core.manager.cluster_manager import ClusterManager +from ainode.core.manager.inference_manager import InferenceManager +from ainode.core.manager.model_manager import ModelManager +from ainode.thrift.ainode import IAINodeRPCService +from ainode.thrift.ainode.ttypes import (TDeleteModelReq, TRegisterModelReq, + TAIHeartbeatReq, TInferenceReq, TRegisterModelResp, TInferenceResp, + TAIHeartbeatResp, TTrainingReq, TForecastReq) +from ainode.thrift.common.ttypes import TSStatus + + +class AINodeRPCServiceHandler(IAINodeRPCService.Iface): + def __init__(self): + self._model_manager = ModelManager() + self._inference_manager = InferenceManager(model_manager=self._model_manager) + + def registerModel(self, req: TRegisterModelReq) -> TRegisterModelResp: + return self._model_manager.register_model(req) + + def deleteModel(self, req: TDeleteModelReq) -> TSStatus: + return self._model_manager.delete_model(req) + + def inference(self, req: TInferenceReq) -> TInferenceResp: + return self._inference_manager.inference(req) + + def forecast(self, req: TForecastReq) -> TSStatus: + return self._inference_manager.forecast(req) + + def getAIHeartbeat(self, req: TAIHeartbeatReq) -> TAIHeartbeatResp: + return ClusterManager.get_heart_beat(req) + + def createTrainingTask(self, req: TTrainingReq) -> TSStatus: + pass diff --git a/iotdb-core/ainode/ainode/core/log.py b/iotdb-core/ainode/ainode/core/log.py new file mode 100644 index 0000000000000..4b2f412eaafb2 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/log.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import inspect +import logging +import multiprocessing +import os +import random +import sys +import threading + +from ainode.core.constant import STD_LEVEL, AINODE_LOG_FILE_NAMES, AINODE_LOG_FILE_LEVELS +from ainode.core.util.decorator import singleton + + +class LoggerFilter(logging.Filter): + def filter(self, record): + record.msg = f"{self.custom_log_info()}: {record.msg}" + return True + + @staticmethod + def custom_log_info(): + frame = inspect.currentframe() + stack_trace = inspect.getouterframes(frame) + + pid = os.getpid() + process_name = multiprocessing.current_process().name + + stack_info = "" + frame_info = stack_trace[7] + file_name = frame_info.filename + # if file_name is not in current working directory, find the first "iotdb" in the path + for l in range(len(file_name)): + i = len(file_name) - l - 1 + if file_name[i:].startswith("iotdb/") or file_name[i:].startswith("iotdb\\"): + file_name = file_name[i:] + break + + stack_info += f"{file_name}:{frame_info.lineno}-{frame_info.function}" + + return f"[{pid}:{process_name}] {stack_info}" + + +@singleton +class Logger: + """ Logger is a singleton, it will be initialized when AINodeDescriptor is inited for the first time. + You can just use Logger() to get it anywhere. + + Args: + log_dir: log directory + + logger_format: log format + logger: global logger with custom format and level + file_handlers: file handlers for different levels + console_handler: console handler for stdout + _lock: process lock for logger. This is just a precaution, we currently do not have multiprocessing + """ + + def __init__(self, log_dir=None): + + self.logger_format = logging.Formatter(fmt='%(asctime)s %(levelname)s %(' + 'message)s', + datefmt='%Y-%m-%d %H:%M:%S') + + self.logger = logging.getLogger(str(random.random())) + self.logger.handlers.clear() + self.logger.setLevel(logging.DEBUG) + self.console_handler = logging.StreamHandler(sys.stdout) + self.console_handler.setLevel(STD_LEVEL) + self.console_handler.setFormatter(self.logger_format) + + self.logger.addHandler(self.console_handler) + + if log_dir is not None: + file_names = AINODE_LOG_FILE_NAMES + file_levels = AINODE_LOG_FILE_LEVELS + if not os.path.exists(log_dir): + os.makedirs(log_dir) + os.chmod(log_dir, 0o777) + for file_name in file_names: + log_path = log_dir + "/" + file_name + if not os.path.exists(log_path): + f = open(log_path, mode='w', encoding='utf-8') + f.close() + os.chmod(log_path, 0o777) + self.file_handlers = [] + for l in range(len(file_names)): + self.file_handlers.append(logging.FileHandler(log_dir + "/" + file_names[l], mode='a')) + self.file_handlers[l].setLevel(file_levels[l]) + self.file_handlers[l].setFormatter(self.logger_format) + + for file_handler in self.file_handlers: + self.logger.addHandler(file_handler) + else: + log_dir = "default path" + + self.logger.addFilter(LoggerFilter()) + self._lock = threading.Lock() + self.info(f"Logger init successfully. Log will be written to {log_dir}") + + def debug(self, *args) -> None: + self._lock.acquire() + self.logger.debug(' '.join(map(str, args))) + self._lock.release() + + def info(self, *args) -> None: + self._lock.acquire() + self.logger.info(' '.join(map(str, args))) + self._lock.release() + + def warning(self, *args) -> None: + self._lock.acquire() + self.logger.warning(' '.join(map(str, args))) + self._lock.release() + + def error(self, *args) -> None: + self._lock.acquire() + self.logger.error(' '.join(map(str, args))) + self._lock.release() diff --git a/iotdb-core/ainode/ainode/core/manager/__init__.py b/iotdb-core/ainode/ainode/core/manager/__init__.py new file mode 100644 index 0000000000000..2a1e720805f29 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/manager/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/iotdb-core/ainode/ainode/core/manager/cluster_manager.py b/iotdb-core/ainode/ainode/core/manager/cluster_manager.py new file mode 100644 index 0000000000000..da7008b776259 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/manager/cluster_manager.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import psutil + +from ainode.thrift.ainode.ttypes import TAIHeartbeatResp, TAIHeartbeatReq +from ainode.thrift.common.ttypes import TLoadSample + + +class ClusterManager: + @staticmethod + def get_heart_beat(req: TAIHeartbeatReq) -> TAIHeartbeatResp: + if req.needSamplingLoad: + cpu_percent = psutil.cpu_percent(interval=1) + memory_percent = psutil.virtual_memory().percent + disk_usage = psutil.disk_usage('/') + disk_free = disk_usage.free + load_sample = TLoadSample(cpuUsageRate=cpu_percent, + memoryUsageRate=memory_percent, + diskUsageRate=disk_usage.percent, + freeDiskSpace=disk_free / 1024 / 1024 / 1024) + return TAIHeartbeatResp(heartbeatTimestamp=req.heartbeatTimestamp, + status="Running", + loadSample=load_sample) + else: + return TAIHeartbeatResp(heartbeatTimestamp=req.heartbeatTimestamp, + status="Running") diff --git a/iotdb-core/ainode/ainode/core/manager/inference_manager.py b/iotdb-core/ainode/ainode/core/manager/inference_manager.py new file mode 100644 index 0000000000000..e5142b0205b67 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/manager/inference_manager.py @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from abc import abstractmethod, ABC + +import pandas as pd +import torch +from iotdb.tsfile.utils.tsblock_serde import deserialize + +from ainode.core.constant import TSStatusCode +from ainode.core.exception import InvalidWindowArgumentError, InferenceModelInternalError, runtime_error_extractor +from ainode.core.log import Logger +from ainode.core.manager.model_manager import ModelManager +from ainode.core.util.serde import convert_to_binary +from ainode.core.util.status import get_status +from ainode.thrift.ainode.ttypes import TInferenceReq, TInferenceResp, TForecastReq, TForecastResp + +logger = Logger() + + +class InferenceStrategy(ABC): + def __init__(self, model): + self.model = model + + @abstractmethod + def infer(self, full_data, **kwargs): + pass + + +# [IoTDB] full data deserialized from iotdb is composed of [timestampList, valueList, length], +# we only get valueList currently. +class TimerXLStrategy(InferenceStrategy): + def infer(self, full_data, predict_length=96, **_): + data = full_data[1][0] + if data.dtype.byteorder not in ('=', '|'): + data = data.byteswap().newbyteorder() + output = self.model.inference(data, int(predict_length)) + df = pd.DataFrame(output[0]) + return convert_to_binary(df) + + +class BuiltInStrategy(InferenceStrategy): + def infer(self, full_data, **_): + data = pd.DataFrame(full_data[1]).T + output = self.model.inference(data) + df = pd.DataFrame(output) + return convert_to_binary(df) + + +class RegisteredStrategy(InferenceStrategy): + def infer(self, full_data, window_interval=None, window_step=None, **kwargs): + _, dataset, _, length = full_data + if window_interval is None or window_step is None: + window_interval = length + window_step = float('inf') + + if window_interval <= 0 or window_step <= 0 or window_interval > length: + raise InvalidWindowArgumentError(window_interval, window_step, length) + + data = torch.tensor(dataset, dtype=torch.float32).unsqueeze(0).permute(0, 2, 1) + + times = int((length - window_interval) // window_step + 1) + results = [] + try: + for i in range(times): + start = 0 if window_step == float('inf') else i * window_step + end = start + window_interval + window = data[:, start:end, :] + out = self.model(window) + df = pd.DataFrame(out.squeeze(0).detach().numpy()) + results.append(df) + except Exception as e: + msg = runtime_error_extractor(str(e)) or str(e) + raise InferenceModelInternalError(msg) + + # concatenate or return first window for forecast + return [convert_to_binary(df) for df in results] + + +def _get_strategy(model_id, model): + if model_id == '_timerxl': + return TimerXLStrategy(model) + if model_id.startswith('_'): + return BuiltInStrategy(model) + return RegisteredStrategy(model) + + +class InferenceManager: + + def __init__(self, model_manager: ModelManager): + self.model_manager = model_manager + + def _run(self, req, data_getter, deserializer, extract_attrs, resp_cls, single_output: bool): + model_id = req.modelId + logger.info(f"Start processing for {model_id}") + try: + raw = data_getter(req) + full_data = deserializer(raw) + attrs = extract_attrs(req) + + # load model + if model_id.startswith('_'): + model = self.model_manager.load_built_in_model(model_id, attrs) + else: + accel = str(attrs.get('acceleration', '')).lower() == 'true' + model = self.model_manager.load_model(model_id, accel) + + # inference by strategy + strategy = _get_strategy(model_id, model) + outputs = strategy.infer(full_data, **attrs) + + # construct response + status = get_status(TSStatusCode.SUCCESS_STATUS) + + if isinstance(outputs, list): + return resp_cls(status, outputs[0] if single_output else outputs) + return resp_cls(status, outputs if single_output else [outputs]) + + except Exception as e: + logger.error(e) + status = get_status(TSStatusCode.AINODE_INTERNAL_ERROR, str(e)) + empty = b'' if single_output else [] + return resp_cls(status, empty) + + def forecast(self, req: TForecastReq): + return self._run( + req, + data_getter=lambda r: r.inputData, + deserializer=deserialize, + extract_attrs=lambda r: {'predict_length': r.outputLength, **(r.options or {})}, + resp_cls=TForecastResp, + single_output=True + ) + + def inference(self, req: TInferenceReq): + return self._run( + req, + data_getter=lambda r: r.dataset, + deserializer=deserialize, + extract_attrs=lambda r: { + 'window_interval': getattr(r.windowParams, 'windowInterval', None), + 'window_step': getattr(r.windowParams, 'windowStep', None), + **(r.inferenceAttributes or {}) + }, + resp_cls=TInferenceResp, + single_output=False + ) diff --git a/iotdb-core/ainode/ainode/core/manager/model_manager.py b/iotdb-core/ainode/ainode/core/manager/model_manager.py new file mode 100644 index 0000000000000..ead833a5839a3 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/manager/model_manager.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from typing import Callable + +from yaml import YAMLError + +from ainode.core.constant import TSStatusCode, BuiltInModelType +from ainode.core.exception import InvalidUriError, BadConfigValueError, BuiltInModelNotSupportError +from ainode.core.log import Logger +from ainode.core.model.built_in_model_factory import fetch_built_in_model +from ainode.core.model.model_storage import ModelStorage +from ainode.core.util.status import get_status +from ainode.thrift.ainode.ttypes import TRegisterModelReq, TRegisterModelResp, TDeleteModelReq +from ainode.thrift.common.ttypes import TSStatus + +logger = Logger() + + +class ModelManager: + def __init__(self): + self.model_storage = ModelStorage() + + def register_model(self, req: TRegisterModelReq) -> TRegisterModelResp: + logger.info(f"register model {req.modelId} from {req.uri}") + try: + configs, attributes = self.model_storage.register_model(req.modelId, req.uri) + return TRegisterModelResp(get_status(TSStatusCode.SUCCESS_STATUS), configs, attributes) + except InvalidUriError as e: + logger.warning(e) + self.model_storage.delete_model(req.modelId) + return TRegisterModelResp(get_status(TSStatusCode.INVALID_URI_ERROR, e.message)) + except BadConfigValueError as e: + logger.warning(e) + self.model_storage.delete_model(req.modelId) + return TRegisterModelResp(get_status(TSStatusCode.INVALID_INFERENCE_CONFIG, e.message)) + except YAMLError as e: + logger.warning(e) + self.model_storage.delete_model(req.modelId) + if hasattr(e, 'problem_mark'): + mark = e.problem_mark + return TRegisterModelResp(get_status(TSStatusCode.INVALID_INFERENCE_CONFIG, + f"An error occurred while parsing the yaml file, " + f"at line {mark.line + 1} column {mark.column + 1}.")) + return TRegisterModelResp( + get_status(TSStatusCode.INVALID_INFERENCE_CONFIG, f"An error occurred while parsing the yaml file")) + except Exception as e: + logger.warning(e) + self.model_storage.delete_model(req.modelId) + return TRegisterModelResp(get_status(TSStatusCode.AINODE_INTERNAL_ERROR)) + + def delete_model(self, req: TDeleteModelReq) -> TSStatus: + logger.info(f"delete model {req.modelId}") + try: + self.model_storage.delete_model(req.modelId) + return get_status(TSStatusCode.SUCCESS_STATUS) + except Exception as e: + logger.warning(e) + return get_status(TSStatusCode.AINODE_INTERNAL_ERROR, str(e)) + + def load_model(self, model_id: str, acceleration: bool = False) -> Callable: + logger.info(f"load model {model_id}") + return self.model_storage.load_model(model_id, acceleration) + + @staticmethod + def load_built_in_model(model_id: str, attributes: {}): + model_id = model_id.lower() + if model_id not in BuiltInModelType.values(): + raise BuiltInModelNotSupportError(model_id) + return fetch_built_in_model(model_id, attributes) diff --git a/iotdb-core/ainode/ainode/core/model/__init__.py b/iotdb-core/ainode/ainode/core/model/__init__.py new file mode 100644 index 0000000000000..2a1e720805f29 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/model/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/iotdb-core/ainode/ainode/core/model/built_in_model_factory.py b/iotdb-core/ainode/ainode/core/model/built_in_model_factory.py new file mode 100644 index 0000000000000..0d2991a7f9f42 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/model/built_in_model_factory.py @@ -0,0 +1,1004 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os +from abc import abstractmethod +from typing import List, Dict + +import numpy as np +from sklearn.preprocessing import MinMaxScaler +from sktime.annotation.hmm_learn import GaussianHMM, GMMHMM +from sktime.annotation.stray import STRAY +from sktime.forecasting.arima import ARIMA +from sktime.forecasting.exp_smoothing import ExponentialSmoothing +from sktime.forecasting.naive import NaiveForecaster +from sktime.forecasting.trend import STLForecaster + +from ainode.TimerXL.models import timer_xl +from ainode.TimerXL.models.configuration_timer import TimerxlConfig +from ainode.core.config import AINodeDescriptor +from ainode.core.constant import AttributeName, BuiltInModelType +from ainode.core.exception import InferenceModelInternalError +from ainode.core.exception import WrongAttributeTypeError, NumericalRangeException, StringRangeException, \ + ListRangeException, BuiltInModelNotSupportError +from ainode.core.log import Logger + +logger = Logger() + + +def get_model_attributes(model_id: str): + if model_id == BuiltInModelType.ARIMA.value: + attribute_map = arima_attribute_map + elif model_id == BuiltInModelType.NAIVE_FORECASTER.value: + attribute_map = naive_forecaster_attribute_map + elif model_id == BuiltInModelType.EXPONENTIAL_SMOOTHING.value: + attribute_map = exponential_smoothing_attribute_map + elif model_id == BuiltInModelType.STL_FORECASTER.value: + attribute_map = stl_forecaster_attribute_map + elif model_id == BuiltInModelType.GMM_HMM.value: + attribute_map = gmmhmm_attribute_map + elif model_id == BuiltInModelType.GAUSSIAN_HMM.value: + attribute_map = gaussian_hmm_attribute_map + elif model_id == BuiltInModelType.STRAY.value: + attribute_map = stray_attribute_map + elif model_id == BuiltInModelType.TIMER_XL.value: + attribute_map = timerxl_attribute_map + else: + raise BuiltInModelNotSupportError(model_id) + return attribute_map + + +def fetch_built_in_model(model_id, inference_attributes): + """ + Args: + model_id: the unique id of the model + inference_attributes: a list of attributes to be inferred, in this function, the attributes will include some + parameters of the built-in model. Some parameters are optional, and if the parameters are not + specified, the default value will be used. + Returns: + model: the built-in model + attributes: a dict of attributes, where the key is the attribute name, the value is the parsed value of the + attribute + Description: + the create_built_in_model function will create the built-in model, which does not require user + registration. This module will parse the inference attributes and create the built-in model. + """ + attribute_map = get_model_attributes(model_id) + + # parse the inference attributes, attributes is a Dict[str, Any] + attributes = parse_attribute(inference_attributes, attribute_map) + + # build the built-in model + if model_id == BuiltInModelType.ARIMA.value: + model = ArimaModel(attributes) + elif model_id == BuiltInModelType.EXPONENTIAL_SMOOTHING.value: + model = ExponentialSmoothingModel(attributes) + elif model_id == BuiltInModelType.NAIVE_FORECASTER.value: + model = NaiveForecasterModel(attributes) + elif model_id == BuiltInModelType.STL_FORECASTER.value: + model = STLForecasterModel(attributes) + elif model_id == BuiltInModelType.GMM_HMM.value: + model = GMMHMMModel(attributes) + elif model_id == BuiltInModelType.GAUSSIAN_HMM.value: + model = GaussianHmmModel(attributes) + elif model_id == BuiltInModelType.STRAY.value: + model = STRAYModel(attributes) + elif model_id == BuiltInModelType.TIMER_XL.value: + model = timer_xl.Model(TimerxlConfig.from_dict(attributes)) + else: + raise BuiltInModelNotSupportError(model_id) + + return model + + +class Attribute(object): + def __init__(self, name: str): + """ + Args: + name: the name of the attribute + """ + self._name = name + + @abstractmethod + def get_default_value(self): + raise NotImplementedError + + @abstractmethod + def validate_value(self, value): + raise NotImplementedError + + @abstractmethod + def parse(self, string_value: str): + raise NotImplementedError + + +class IntAttribute(Attribute): + def __init__(self, name: str, + default_value: int, + default_low: int, + default_high: int, + ): + super(IntAttribute, self).__init__(name) + self.__default_value = default_value + self.__default_low = default_low + self.__default_high = default_high + + def get_default_value(self): + return self.__default_value + + def validate_value(self, value): + if self.__default_low <= value <= self.__default_high: + return True + raise NumericalRangeException(self._name, value, self.__default_low, self.__default_high) + + def parse(self, string_value: str): + try: + int_value = int(string_value) + except: + raise WrongAttributeTypeError(self._name, "int") + return int_value + + +class FloatAttribute(Attribute): + def __init__(self, name: str, + default_value: float, + default_low: float, + default_high: float, + ): + super(FloatAttribute, self).__init__(name) + self.__default_value = default_value + self.__default_low = default_low + self.__default_high = default_high + + def get_default_value(self): + return self.__default_value + + def validate_value(self, value): + if self.__default_low <= value <= self.__default_high: + return True + raise NumericalRangeException(self._name, value, self.__default_low, self.__default_high) + + def parse(self, string_value: str): + try: + float_value = float(string_value) + except: + raise WrongAttributeTypeError(self._name, "float") + return float_value + + +class StringAttribute(Attribute): + def __init__(self, name: str, default_value: str, value_choices: List[str]): + super(StringAttribute, self).__init__(name) + self.__default_value = default_value + self.__value_choices = value_choices + + def get_default_value(self): + return self.__default_value + + def validate_value(self, value): + if value in self.__value_choices: + return True + raise StringRangeException(self._name, value, self.__value_choices) + + def parse(self, string_value: str): + return string_value + + +class BooleanAttribute(Attribute): + def __init__(self, name: str, default_value: bool): + super(BooleanAttribute, self).__init__(name) + self.__default_value = default_value + + def get_default_value(self): + return self.__default_value + + def validate_value(self, value): + if isinstance(value, bool): + return True + raise WrongAttributeTypeError(self._name, "bool") + + def parse(self, string_value: str): + if string_value.lower() == "true": + return True + elif string_value.lower() == "false": + return False + else: + raise WrongAttributeTypeError(self._name, "bool") + + +class ListAttribute(Attribute): + def __init__(self, name: str, default_value: List, value_type): + """ + value_type is the type of the elements in the list, e.g. int, float, str + """ + super(ListAttribute, self).__init__(name) + self.__default_value = default_value + self.__value_type = value_type + self.__type_to_str = {str: "str", int: "int", float: "float"} + + def get_default_value(self): + return self.__default_value + + def validate_value(self, value): + if not isinstance(value, list): + raise WrongAttributeTypeError(self._name, "list") + for value_item in value: + if not isinstance(value_item, self.__value_type): + raise WrongAttributeTypeError(self._name, self.__value_type) + return True + + def parse(self, string_value: str): + try: + list_value = eval(string_value) + except: + raise WrongAttributeTypeError(self._name, "list") + if not isinstance(list_value, list): + raise WrongAttributeTypeError(self._name, "list") + for i in range(len(list_value)): + try: + list_value[i] = self.__value_type(list_value[i]) + except: + raise ListRangeException(self._name, list_value, self.__type_to_str[self.__value_type]) + return list_value + + +class TupleAttribute(Attribute): + def __init__(self, name: str, default_value: tuple, value_type): + """ + value_type is the type of the elements in the list, e.g. int, float, str + """ + super(TupleAttribute, self).__init__(name) + self.__default_value = default_value + self.__value_type = value_type + self.__type_to_str = {str: "str", int: "int", float: "float"} + + def get_default_value(self): + return self.__default_value + + def validate_value(self, value): + if not isinstance(value, tuple): + raise WrongAttributeTypeError(self._name, "tuple") + for value_item in value: + if not isinstance(value_item, self.__value_type): + raise WrongAttributeTypeError(self._name, self.__value_type) + return True + + def parse(self, string_value: str): + try: + tuple_value = eval(string_value) + except: + raise WrongAttributeTypeError(self._name, "tuple") + if not isinstance(tuple_value, tuple): + raise WrongAttributeTypeError(self._name, "tuple") + list_value = list(tuple_value) + for i in range(len(list_value)): + try: + list_value[i] = self.__value_type(list_value[i]) + except: + raise ListRangeException(self._name, list_value, self.__type_to_str[self.__value_type]) + tuple_value = tuple(list_value) + return tuple_value + + +def parse_attribute(input_attributes: Dict[str, str], attribute_map: Dict[str, Attribute]): + """ + Args: + input_attributes: a dict of attributes, where the key is the attribute name, the value is the string value of + the attribute + attribute_map: a dict of hyperparameters, where the key is the attribute name, the value is the Attribute + object + Returns: + a dict of attributes, where the key is the attribute name, the value is the parsed value of the attribute + """ + attributes = {} + for attribute_name in attribute_map: + # user specified the attribute + if attribute_name in input_attributes: + attribute = attribute_map[attribute_name] + value = attribute.parse(input_attributes[attribute_name]) + attribute.validate_value(value) + attributes[attribute_name] = value + # user did not specify the attribute, use the default value + else: + try: + attributes[attribute_name] = attribute_map[attribute_name].get_default_value() + except NotImplementedError as e: + logger.error(f"attribute {attribute_name} is not implemented.") + raise e + return attributes + + +timerxl_attribute_map = { + AttributeName.INPUT_TOKEN_LEN.value: IntAttribute( + name=AttributeName.INPUT_TOKEN_LEN.value, + default_value=96, + default_low=1, + default_high=5000 + ), + AttributeName.HIDDEN_SIZE.value: IntAttribute( + name=AttributeName.HIDDEN_SIZE.value, + default_value=1024, + default_low=1, + default_high=5000 + ), + AttributeName.INTERMEDIATE_SIZE.value: IntAttribute( + name=AttributeName.INTERMEDIATE_SIZE.value, + default_value=2048, + default_low=1, + default_high=5000 + ), + AttributeName.OUTPUT_TOKEN_LENS.value: ListAttribute( + name=AttributeName.OUTPUT_TOKEN_LENS.value, + default_value=[96], + value_type=int + ), + AttributeName.NUM_HIDDEN_LAYERS.value: IntAttribute( + name=AttributeName.NUM_HIDDEN_LAYERS.value, + default_value=8, + default_low=1, + default_high=16 + ), + AttributeName.NUM_ATTENTION_HEADS.value: IntAttribute( + name=AttributeName.NUM_ATTENTION_HEADS.value, + default_value=8, + default_low=1, + default_high=192 + ), + AttributeName.HIDDEN_ACT.value: StringAttribute( + name=AttributeName.HIDDEN_ACT.value, + default_value="silu", + value_choices=["relu", "gelu", "silu", "tanh"], + ), + AttributeName.USE_CACHE.value: BooleanAttribute( + name=AttributeName.USE_CACHE.value, + default_value=True, + ), + AttributeName.ROPE_THETA.value: IntAttribute( + name=AttributeName.ROPE_THETA.value, + default_value=10000, + default_low=1000, + default_high=50000 + ), + AttributeName.ATTENTION_DROPOUT.value: FloatAttribute( + name=AttributeName.ATTENTION_DROPOUT.value, + default_value=0.0, + default_low=0.0, + default_high=1.0 + ), + AttributeName.INITIALIZER_RANGE.value: FloatAttribute( + name=AttributeName.INITIALIZER_RANGE.value, + default_value=0.02, + default_low=0.0, + default_high=1.0 + ), + AttributeName.MAX_POSITION_EMBEDDINGS.value: IntAttribute( + name=AttributeName.MAX_POSITION_EMBEDDINGS.value, + default_value=10000, + default_low=1, + default_high=50000 + ), + AttributeName.TIMERXL_CKPT_PATH.value: StringAttribute( + name=AttributeName.TIMERXL_CKPT_PATH.value, + default_value=os.path.join(os.getcwd(), AINodeDescriptor().get_config().get_ain_models_dir(), 'weights', + 'timerxl', 'model.safetensors'), + value_choices=[''] + ) +} + +# built-in sktime model attributes +# NaiveForecaster +naive_forecaster_attribute_map = { + AttributeName.PREDICT_LENGTH.value: IntAttribute( + name=AttributeName.PREDICT_LENGTH.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.STRATEGY.value: StringAttribute( + name=AttributeName.STRATEGY.value, + default_value="last", + value_choices=["last", "mean"], + ), + AttributeName.SP.value: IntAttribute( + name=AttributeName.SP.value, + default_value=1, + default_low=1, + default_high=5000 + ), +} +# ExponentialSmoothing +exponential_smoothing_attribute_map = { + AttributeName.PREDICT_LENGTH.value: IntAttribute( + name=AttributeName.PREDICT_LENGTH.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.DAMPED_TREND.value: BooleanAttribute( + name=AttributeName.DAMPED_TREND.value, + default_value=False, + ), + AttributeName.INITIALIZATION_METHOD.value: StringAttribute( + name=AttributeName.INITIALIZATION_METHOD.value, + default_value="estimated", + value_choices=["estimated", "heuristic", "legacy-heuristic", "known"], + ), + AttributeName.OPTIMIZED.value: BooleanAttribute( + name=AttributeName.OPTIMIZED.value, + default_value=True, + ), + AttributeName.REMOVE_BIAS.value: BooleanAttribute( + name=AttributeName.REMOVE_BIAS.value, + default_value=False, + ), + AttributeName.USE_BRUTE.value: BooleanAttribute( + name=AttributeName.USE_BRUTE.value, + default_value=False, + ) +} +# Arima +arima_attribute_map = { + AttributeName.PREDICT_LENGTH.value: IntAttribute( + name=AttributeName.PREDICT_LENGTH.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.ORDER.value: TupleAttribute( + name=AttributeName.ORDER.value, + default_value=(1, 0, 0), + value_type=int + ), + AttributeName.SEASONAL_ORDER.value: TupleAttribute( + name=AttributeName.SEASONAL_ORDER.value, + default_value=(0, 0, 0, 0), + value_type=int + ), + AttributeName.METHOD.value: StringAttribute( + name=AttributeName.METHOD.value, + default_value="lbfgs", + value_choices=["lbfgs", "bfgs", "newton", "nm", "cg", "ncg", "powell"], + ), + AttributeName.MAXITER.value: IntAttribute( + name=AttributeName.MAXITER.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.SUPPRESS_WARNINGS.value: BooleanAttribute( + name=AttributeName.SUPPRESS_WARNINGS.value, + default_value=True, + ), + AttributeName.OUT_OF_SAMPLE_SIZE.value: IntAttribute( + name=AttributeName.OUT_OF_SAMPLE_SIZE.value, + default_value=0, + default_low=0, + default_high=5000 + ), + AttributeName.SCORING.value: StringAttribute( + name=AttributeName.SCORING.value, + default_value="mse", + value_choices=["mse", "mae", "rmse", "mape", "smape", "rmsle", "r2"], + ), + AttributeName.WITH_INTERCEPT.value: BooleanAttribute( + name=AttributeName.WITH_INTERCEPT.value, + default_value=True, + ), + AttributeName.TIME_VARYING_REGRESSION.value: BooleanAttribute( + name=AttributeName.TIME_VARYING_REGRESSION.value, + default_value=False, + ), + AttributeName.ENFORCE_STATIONARITY.value: BooleanAttribute( + name=AttributeName.ENFORCE_STATIONARITY.value, + default_value=True, + ), + AttributeName.ENFORCE_INVERTIBILITY.value: BooleanAttribute( + name=AttributeName.ENFORCE_INVERTIBILITY.value, + default_value=True, + ), + AttributeName.SIMPLE_DIFFERENCING.value: BooleanAttribute( + name=AttributeName.SIMPLE_DIFFERENCING.value, + default_value=False, + ), + AttributeName.MEASUREMENT_ERROR.value: BooleanAttribute( + name=AttributeName.MEASUREMENT_ERROR.value, + default_value=False, + ), + AttributeName.MLE_REGRESSION.value: BooleanAttribute( + name=AttributeName.MLE_REGRESSION.value, + default_value=True, + ), + AttributeName.HAMILTON_REPRESENTATION.value: BooleanAttribute( + name=AttributeName.HAMILTON_REPRESENTATION.value, + default_value=False, + ), + AttributeName.CONCENTRATE_SCALE.value: BooleanAttribute( + name=AttributeName.CONCENTRATE_SCALE.value, + default_value=False, + ) +} +# STLForecaster +stl_forecaster_attribute_map = { + AttributeName.PREDICT_LENGTH.value: IntAttribute( + name=AttributeName.PREDICT_LENGTH.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.SP.value: IntAttribute( + name=AttributeName.SP.value, + default_value=2, + default_low=1, + default_high=5000 + ), + AttributeName.SEASONAL.value: IntAttribute( + name=AttributeName.SEASONAL.value, + default_value=7, + default_low=1, + default_high=5000 + ), + AttributeName.SEASONAL_DEG.value: IntAttribute( + name=AttributeName.SEASONAL_DEG.value, + default_value=1, + default_low=0, + default_high=5000 + ), + AttributeName.TREND_DEG.value: IntAttribute( + name=AttributeName.TREND_DEG.value, + default_value=1, + default_low=0, + default_high=5000 + ), + AttributeName.LOW_PASS_DEG.value: IntAttribute( + name=AttributeName.LOW_PASS_DEG.value, + default_value=1, + default_low=0, + default_high=5000 + ), + AttributeName.SEASONAL_JUMP.value: IntAttribute( + name=AttributeName.SEASONAL_JUMP.value, + default_value=1, + default_low=0, + default_high=5000 + ), + AttributeName.TREND_JUMP.value: IntAttribute( + name=AttributeName.TREND_JUMP.value, + default_value=1, + default_low=0, + default_high=5000 + ), + AttributeName.LOSS_PASS_JUMP.value: IntAttribute( + name=AttributeName.LOSS_PASS_JUMP.value, + default_value=1, + default_low=0, + default_high=5000 + ), +} + +# GAUSSIAN_HMM +gaussian_hmm_attribute_map = { + AttributeName.N_COMPONENTS.value: IntAttribute( + name=AttributeName.N_COMPONENTS.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.COVARIANCE_TYPE.value: StringAttribute( + name=AttributeName.COVARIANCE_TYPE.value, + default_value="diag", + value_choices=["spherical", "diag", "full", "tied"], + ), + AttributeName.MIN_COVAR.value: FloatAttribute( + name=AttributeName.MIN_COVAR.value, + default_value=1e-3, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.STARTPROB_PRIOR.value: FloatAttribute( + name=AttributeName.STARTPROB_PRIOR.value, + default_value=1.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.TRANSMAT_PRIOR.value: FloatAttribute( + name=AttributeName.TRANSMAT_PRIOR.value, + default_value=1.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.MEANS_PRIOR.value: FloatAttribute( + name=AttributeName.MEANS_PRIOR.value, + default_value=0.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.MEANS_WEIGHT.value: FloatAttribute( + name=AttributeName.MEANS_WEIGHT.value, + default_value=0.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.COVARS_PRIOR.value: FloatAttribute( + name=AttributeName.COVARS_PRIOR.value, + default_value=1e-2, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.COVARS_WEIGHT.value: FloatAttribute( + name=AttributeName.COVARS_WEIGHT.value, + default_value=1.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.ALGORITHM.value: StringAttribute( + name=AttributeName.ALGORITHM.value, + default_value="viterbi", + value_choices=["viterbi", "map"], + ), + AttributeName.N_ITER.value: IntAttribute( + name=AttributeName.N_ITER.value, + default_value=10, + default_low=1, + default_high=5000 + ), + AttributeName.TOL.value: FloatAttribute( + name=AttributeName.TOL.value, + default_value=1e-2, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.PARAMS.value: StringAttribute( + name=AttributeName.PARAMS.value, + default_value="stmc", + value_choices=["stmc", "stm"], + ), + AttributeName.INIT_PARAMS.value: StringAttribute( + name=AttributeName.INIT_PARAMS.value, + default_value="stmc", + value_choices=["stmc", "stm"], + ), + AttributeName.IMPLEMENTATION.value: StringAttribute( + name=AttributeName.IMPLEMENTATION.value, + default_value="log", + value_choices=["log", "scaling"], + ) +} + +# GMMHMM +gmmhmm_attribute_map = { + AttributeName.N_COMPONENTS.value: IntAttribute( + name=AttributeName.N_COMPONENTS.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.N_MIX.value: IntAttribute( + name=AttributeName.N_MIX.value, + default_value=1, + default_low=1, + default_high=5000 + ), + AttributeName.MIN_COVAR.value: FloatAttribute( + name=AttributeName.MIN_COVAR.value, + default_value=1e-3, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.STARTPROB_PRIOR.value: FloatAttribute( + name=AttributeName.STARTPROB_PRIOR.value, + default_value=1.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.TRANSMAT_PRIOR.value: FloatAttribute( + name=AttributeName.TRANSMAT_PRIOR.value, + default_value=1.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.WEIGHTS_PRIOR.value: FloatAttribute( + name=AttributeName.WEIGHTS_PRIOR.value, + default_value=1.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.MEANS_PRIOR.value: FloatAttribute( + name=AttributeName.MEANS_PRIOR.value, + default_value=0.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.MEANS_WEIGHT.value: FloatAttribute( + name=AttributeName.MEANS_WEIGHT.value, + default_value=0.0, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.ALGORITHM.value: StringAttribute( + name=AttributeName.ALGORITHM.value, + default_value="viterbi", + value_choices=["viterbi", "map"], + ), + AttributeName.COVARIANCE_TYPE.value: StringAttribute( + name=AttributeName.COVARIANCE_TYPE.value, + default_value="diag", + value_choices=["sperical", "diag", "full", "tied"], + ), + AttributeName.N_ITER.value: IntAttribute( + name=AttributeName.N_ITER.value, + default_value=10, + default_low=1, + default_high=5000 + ), + AttributeName.TOL.value: FloatAttribute( + name=AttributeName.TOL.value, + default_value=1e-2, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.INIT_PARAMS.value: StringAttribute( + name=AttributeName.INIT_PARAMS.value, + default_value="stmcw", + value_choices=["s", "t", "m", "c", "w", "st", "sm", "sc", "sw", "tm", "tc", "tw", "mc", "mw", "cw", "stm", + "stc", "stw", "smc", "smw", "scw", "tmc", "tmw", "tcw", "mcw", "stmc", "stmw", "stcw", "smcw", + "tmcw", "stmcw"] + ), + AttributeName.PARAMS.value: StringAttribute( + name=AttributeName.PARAMS.value, + default_value="stmcw", + value_choices=["s", "t", "m", "c", "w", "st", "sm", "sc", "sw", "tm", "tc", "tw", "mc", "mw", "cw", "stm", + "stc", "stw", "smc", "smw", "scw", "tmc", "tmw", "tcw", "mcw", "stmc", "stmw", "stcw", "smcw", + "tmcw", "stmcw"] + ), + AttributeName.IMPLEMENTATION.value: StringAttribute( + name=AttributeName.IMPLEMENTATION.value, + default_value="log", + value_choices=["log", "scaling"], + ) +} + +# STRAY +stray_attribute_map = { + AttributeName.ALPHA.value: FloatAttribute( + name=AttributeName.ALPHA.value, + default_value=0.01, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.K.value: IntAttribute( + name=AttributeName.K.value, + default_value=10, + default_low=1, + default_high=5000 + ), + AttributeName.KNN_ALGORITHM.value: StringAttribute( + name=AttributeName.KNN_ALGORITHM.value, + default_value="brute", + value_choices=["brute", "kd_tree", "ball_tree", "auto"], + ), + AttributeName.P.value: FloatAttribute( + name=AttributeName.P.value, + default_value=0.5, + default_low=-1e10, + default_high=1e10, + ), + AttributeName.SIZE_THRESHOLD.value: IntAttribute( + name=AttributeName.SIZE_THRESHOLD.value, + default_value=50, + default_low=1, + default_high=5000 + ), + AttributeName.OUTLIER_TAIL.value: StringAttribute( + name=AttributeName.OUTLIER_TAIL.value, + default_value="max", + value_choices=["min", "max"], + ), +} + + +class BuiltInModel(object): + def __init__(self, attributes): + self._attributes = attributes + self._model = None + + @abstractmethod + def inference(self, data): + raise NotImplementedError + + +class ArimaModel(BuiltInModel): + def __init__(self, attributes): + super(ArimaModel, self).__init__(attributes) + self._model = ARIMA( + order=attributes['order'], + seasonal_order=attributes['seasonal_order'], + method=attributes['method'], + suppress_warnings=attributes['suppress_warnings'], + maxiter=attributes['maxiter'], + out_of_sample_size=attributes['out_of_sample_size'], + scoring=attributes['scoring'], + with_intercept=attributes['with_intercept'], + time_varying_regression=attributes['time_varying_regression'], + enforce_stationarity=attributes['enforce_stationarity'], + enforce_invertibility=attributes['enforce_invertibility'], + simple_differencing=attributes['simple_differencing'], + measurement_error=attributes['measurement_error'], + mle_regression=attributes['mle_regression'], + hamilton_representation=attributes['hamilton_representation'], + concentrate_scale=attributes['concentrate_scale'] + ) + + def inference(self, data): + try: + predict_length = self._attributes['predict_length'] + self._model.fit(data) + output = self._model.predict(fh=range(predict_length)) + output = np.array(output, dtype=np.float64) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) + + +class ExponentialSmoothingModel(BuiltInModel): + def __init__(self, attributes): + super(ExponentialSmoothingModel, self).__init__(attributes) + self._model = ExponentialSmoothing( + damped_trend=attributes['damped_trend'], + initialization_method=attributes['initialization_method'], + optimized=attributes['optimized'], + remove_bias=attributes['remove_bias'], + use_brute=attributes['use_brute'] + ) + + def inference(self, data): + try: + predict_length = self._attributes['predict_length'] + self._model.fit(data) + output = self._model.predict(fh=range(predict_length)) + output = np.array(output, dtype=np.float64) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) + + +class NaiveForecasterModel(BuiltInModel): + def __init__(self, attributes): + super(NaiveForecasterModel, self).__init__(attributes) + self._model = NaiveForecaster( + strategy=attributes['strategy'], + sp=attributes['sp'] + ) + + def inference(self, data): + try: + predict_length = self._attributes['predict_length'] + self._model.fit(data) + output = self._model.predict(fh=range(predict_length)) + output = np.array(output, dtype=np.float64) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) + + +class STLForecasterModel(BuiltInModel): + def __init__(self, attributes): + super(STLForecasterModel, self).__init__(attributes) + self._model = STLForecaster( + sp=attributes['sp'], + seasonal=attributes['seasonal'], + seasonal_deg=attributes['seasonal_deg'], + trend_deg=attributes['trend_deg'], + low_pass_deg=attributes['low_pass_deg'], + seasonal_jump=attributes['seasonal_jump'], + trend_jump=attributes['trend_jump'], + low_pass_jump=attributes['low_pass_jump'] + ) + + def inference(self, data): + try: + predict_length = self._attributes['predict_length'] + self._model.fit(data) + output = self._model.predict(fh=range(predict_length)) + output = np.array(output, dtype=np.float64) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) + + +class GMMHMMModel(BuiltInModel): + def __init__(self, attributes): + super(GMMHMMModel, self).__init__(attributes) + self._model = GMMHMM( + n_components=attributes['n_components'], + n_mix=attributes['n_mix'], + min_covar=attributes['min_covar'], + startprob_prior=attributes['startprob_prior'], + transmat_prior=attributes['transmat_prior'], + means_prior=attributes['means_prior'], + means_weight=attributes['means_weight'], + weights_prior=attributes['weights_prior'], + algorithm=attributes['algorithm'], + covariance_type=attributes['covariance_type'], + n_iter=attributes['n_iter'], + tol=attributes['tol'], + params=attributes['params'], + init_params=attributes['init_params'], + implementation=attributes['implementation'] + ) + + def inference(self, data): + try: + self._model.fit(data) + output = self._model.predict(data) + output = np.array(output, dtype=np.int32) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) + + +class GaussianHmmModel(BuiltInModel): + def __init__(self, attributes): + super(GaussianHmmModel, self).__init__(attributes) + self._model = GaussianHMM( + n_components=attributes['n_components'], + covariance_type=attributes['covariance_type'], + min_covar=attributes['min_covar'], + startprob_prior=attributes['startprob_prior'], + transmat_prior=attributes['transmat_prior'], + means_prior=attributes['means_prior'], + means_weight=attributes['means_weight'], + covars_prior=attributes['covars_prior'], + covars_weight=attributes['covars_weight'], + algorithm=attributes['algorithm'], + n_iter=attributes['n_iter'], + tol=attributes['tol'], + params=attributes['params'], + init_params=attributes['init_params'], + implementation=attributes['implementation'] + ) + + def inference(self, data): + try: + self._model.fit(data) + output = self._model.predict(data) + output = np.array(output, dtype=np.int32) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) + + +class STRAYModel(BuiltInModel): + def __init__(self, attributes): + super(STRAYModel, self).__init__(attributes) + self._model = STRAY( + alpha=attributes['alpha'], + k=attributes['k'], + knn_algorithm=attributes['knn_algorithm'], + p=attributes['p'], + size_threshold=attributes['size_threshold'], + outlier_tail=attributes['outlier_tail'] + ) + + def inference(self, data): + try: + data = MinMaxScaler().fit_transform(data) + output = self._model.fit_transform(data) + # change the output to int + output = np.array(output, dtype=np.int32) + return output + except Exception as e: + raise InferenceModelInternalError(str(e)) diff --git a/iotdb-core/ainode/ainode/core/model/model_factory.py b/iotdb-core/ainode/ainode/core/model/model_factory.py new file mode 100644 index 0000000000000..1700dd28eb64a --- /dev/null +++ b/iotdb-core/ainode/ainode/core/model/model_factory.py @@ -0,0 +1,235 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os +import shutil +from urllib.parse import urlparse, urljoin + +import yaml +from requests import Session +from requests.adapters import HTTPAdapter + +from ainode.core.constant import DEFAULT_RECONNECT_TIMES, DEFAULT_RECONNECT_TIMEOUT, DEFAULT_CHUNK_SIZE, \ + DEFAULT_CONFIG_FILE_NAME, DEFAULT_MODEL_FILE_NAME +from ainode.core.exception import InvalidUriError, BadConfigValueError +from ainode.core.log import Logger +from ainode.core.util.serde import get_data_type_byte_from_str +from ainode.thrift.ainode.ttypes import TConfigs + +HTTP_PREFIX = "http://" +HTTPS_PREFIX = "https://" + +logger = Logger() + + +def _parse_uri(uri): + """ + Args: + uri (str): uri to parse + Returns: + is_network_path (bool): True if the url is a network path, False otherwise + parsed_uri (str): parsed uri to get related file + """ + + parse_result = urlparse(uri) + is_network_path = parse_result.scheme in ('http', 'https') + if is_network_path: + return True, uri + + # handle file:// in uri + if parse_result.scheme == 'file': + uri = uri[7:] + + # handle ~ in uri + uri = os.path.expanduser(uri) + return False, uri + + +def _download_file(url: str, storage_path: str) -> None: + """ + Args: + url: url of file to download + storage_path: path to save the file + Returns: + None + """ + logger.debug(f"download file from {url} to {storage_path}") + + session = Session() + adapter = HTTPAdapter(max_retries=DEFAULT_RECONNECT_TIMES) + session.mount(HTTP_PREFIX, adapter) + session.mount(HTTPS_PREFIX, adapter) + + response = session.get(url, timeout=DEFAULT_RECONNECT_TIMEOUT, stream=True) + response.raise_for_status() + + with open(storage_path, 'wb') as file: + for chunk in response.iter_content(chunk_size=DEFAULT_CHUNK_SIZE): + if chunk: + file.write(chunk) + + logger.debug(f"download file from {url} to {storage_path} success") + + +def _register_model_from_network(uri: str, model_storage_path: str, + config_storage_path: str) -> [TConfigs, str]: + """ + Args: + uri: network dir path of model to register, where model.pt and config.yaml are required, + e.g. https://huggingface.co/user/modelname/resolve/main/ + model_storage_path: path to save model.pt + config_storage_path: path to save config.yaml + Returns: + configs: TConfigs + attributes: str + """ + # concat uri to get complete url + uri = uri if uri.endswith("/") else uri + "/" + target_model_path = urljoin(uri, DEFAULT_MODEL_FILE_NAME) + target_config_path = urljoin(uri, DEFAULT_CONFIG_FILE_NAME) + + # download config file + _download_file(target_config_path, config_storage_path) + + # read and parse config dict from config.yaml + with open(config_storage_path, 'r', encoding='utf-8') as file: + config_dict = yaml.safe_load(file) + configs, attributes = _parse_inference_config(config_dict) + + # if config.yaml is correct, download model file + _download_file(target_model_path, model_storage_path) + return configs, attributes + + +def _register_model_from_local(uri: str, model_storage_path: str, + config_storage_path: str) -> [TConfigs, str]: + """ + Args: + uri: local dir path of model to register, where model.pt and config.yaml are required, + e.g. /Users/admin/Desktop/model + model_storage_path: path to save model.pt + config_storage_path: path to save config.yaml + Returns: + configs: TConfigs + attributes: str + """ + # concat uri to get complete path + target_model_path = os.path.join(uri, DEFAULT_MODEL_FILE_NAME) + target_config_path = os.path.join(uri, DEFAULT_CONFIG_FILE_NAME) + + # check if file exist + exist_model_file = os.path.exists(target_model_path) + exist_config_file = os.path.exists(target_config_path) + + configs = None + attributes = None + if exist_model_file and exist_config_file: + # copy config.yaml + logger.debug(f"copy file from {target_config_path} to {config_storage_path}") + shutil.copy(target_config_path, config_storage_path) + logger.debug(f"copy file from {target_config_path} to {config_storage_path} success") + + # read and parse config dict from config.yaml + with open(config_storage_path, 'r', encoding='utf-8') as file: + config_dict = yaml.safe_load(file) + configs, attributes = _parse_inference_config(config_dict) + + # if config.yaml is correct, copy model file + logger.debug(f"copy file from {target_model_path} to {model_storage_path}") + shutil.copy(target_model_path, model_storage_path) + logger.debug(f"copy file from {target_model_path} to {model_storage_path} success") + + elif not exist_model_file or not exist_config_file: + raise InvalidUriError(uri) + + return configs, attributes + + +def _parse_inference_config(config_dict): + """ + Args: + config_dict: dict + - configs: dict + - input_shape (list): input shape of the model and needs to be two-dimensional array like [96, 2] + - output_shape (list): output shape of the model and needs to be two-dimensional array like [96, 2] + - input_type (list): input type of the model and each element needs to be in ['bool', 'int32', 'int64', 'float32', 'float64', 'text'], default float64 + - output_type (list): output type of the model and each element needs to be in ['bool', 'int32', 'int64', 'float32', 'float64', 'text'], default float64 + - attributes: dict + Returns: + configs: TConfigs + attributes: str + """ + configs = config_dict['configs'] + + # check if input_shape and output_shape are two-dimensional array + if not (isinstance(configs['input_shape'], list) and len(configs['input_shape']) == 2): + raise BadConfigValueError('input_shape', configs['input_shape'], + 'input_shape should be a two-dimensional array.') + if not (isinstance(configs['output_shape'], list) and len(configs['output_shape']) == 2): + raise BadConfigValueError('output_shape', configs['output_shape'], + 'output_shape should be a two-dimensional array.') + + # check if input_shape and output_shape are positive integer + input_shape_is_positive_number = isinstance(configs['input_shape'][0], int) and isinstance( + configs['input_shape'][1], int) and configs['input_shape'][0] > 0 and configs['input_shape'][1] > 0 + if not input_shape_is_positive_number: + raise BadConfigValueError('input_shape', configs['input_shape'], + 'element in input_shape should be positive integer.') + + output_shape_is_positive_number = isinstance(configs['output_shape'][0], int) and isinstance( + configs['output_shape'][1], int) and configs['output_shape'][0] > 0 and configs['output_shape'][1] > 0 + if not output_shape_is_positive_number: + raise BadConfigValueError('output_shape', configs['output_shape'], + 'element in output_shape should be positive integer.') + + # check if input_type and output_type are one-dimensional array with right length + if 'input_type' in configs and not ( + isinstance(configs['input_type'], list) and len(configs['input_type']) == configs['input_shape'][1]): + raise BadConfigValueError('input_type', configs['input_type'], + 'input_type should be a one-dimensional array and length of it should be equal to input_shape[1].') + + if 'output_type' in configs and not ( + isinstance(configs['output_type'], list) and len(configs['output_type']) == configs['output_shape'][1]): + raise BadConfigValueError('output_type', configs['output_type'], + 'output_type should be a one-dimensional array and length of it should be equal to output_shape[1].') + + # parse input_type and output_type to byte + if 'input_type' in configs: + input_type = [get_data_type_byte_from_str(x) for x in configs['input_type']] + else: + input_type = [get_data_type_byte_from_str('float32')] * configs['input_shape'][1] + + if 'output_type' in configs: + output_type = [get_data_type_byte_from_str(x) for x in configs['output_type']] + else: + output_type = [get_data_type_byte_from_str('float32')] * configs['output_shape'][1] + + # parse attributes + attributes = "" + if 'attributes' in config_dict: + attributes = str(config_dict['attributes']) + + return TConfigs(configs['input_shape'], configs['output_shape'], input_type, output_type), attributes + + +def fetch_model_by_uri(uri: str, model_storage_path: str, config_storage_path: str): + is_network_path, uri = _parse_uri(uri) + + if is_network_path: + return _register_model_from_network(uri, model_storage_path, config_storage_path) + else: + return _register_model_from_local(uri, model_storage_path, config_storage_path) diff --git a/iotdb-core/ainode/ainode/core/model/model_storage.py b/iotdb-core/ainode/ainode/core/model/model_storage.py new file mode 100644 index 0000000000000..43ebf6c06b796 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/model/model_storage.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import os +import shutil +from collections.abc import Callable + +import torch +import torch._dynamo +from pylru import lrucache + +from ainode.core.config import AINodeDescriptor +from ainode.core.constant import (DEFAULT_MODEL_FILE_NAME, + DEFAULT_CONFIG_FILE_NAME) +from ainode.core.exception import ModelNotExistError +from ainode.core.log import Logger +from ainode.core.model.model_factory import fetch_model_by_uri +from ainode.core.util.lock import ModelLockPool +logger = Logger() + + +class ModelStorage(object): + def __init__(self): + self._model_dir = os.path.join(os.getcwd(), AINodeDescriptor().get_config().get_ain_models_dir()) + if not os.path.exists(self._model_dir): + try: + os.makedirs(self._model_dir) + except PermissionError as e: + logger.error(e) + raise e + self._lock_pool = ModelLockPool() + self._model_cache = lrucache(AINodeDescriptor().get_config().get_ain_model_storage_cache_size()) + + def register_model(self, model_id: str, uri: str): + """ + Args: + model_id: id of model to register + uri: network dir path or local dir path of model to register, where model.pt and config.yaml are required, + e.g. https://huggingface.co/user/modelname/resolve/main/ or /Users/admin/Desktop/model + Returns: + configs: TConfigs + attributes: str + """ + storage_path = os.path.join(self._model_dir, f'{model_id}') + # create storage dir if not exist + if not os.path.exists(storage_path): + os.makedirs(storage_path) + model_storage_path = os.path.join(storage_path, DEFAULT_MODEL_FILE_NAME) + config_storage_path = os.path.join(storage_path, DEFAULT_CONFIG_FILE_NAME) + return fetch_model_by_uri(uri, model_storage_path, config_storage_path) + + def load_model(self, model_id: str, acceleration: bool) -> Callable: + """ + Returns: + model: a ScriptModule contains model architecture and parameters, which can be deployed cross-platform + """ + ain_models_dir = os.path.join(self._model_dir, f'{model_id}') + model_path = os.path.join(ain_models_dir, DEFAULT_MODEL_FILE_NAME) + with self._lock_pool.get_lock(model_id).read_lock(): + if model_path in self._model_cache: + model = self._model_cache[model_path] + if isinstance(model, torch._dynamo.eval_frame.OptimizedModule) or not acceleration: + return model + else: + model = torch.compile(model) + self._model_cache[model_path] = model + return model + else: + if not os.path.exists(model_path): + raise ModelNotExistError(model_path) + else: + model = torch.jit.load(model_path) + if acceleration: + try: + model = torch.compile(model) + except Exception as e: + logger.warning(f"acceleration failed, fallback to normal mode: {str(e)}") + self._model_cache[model_path] = model + return model + + def delete_model(self, model_id: str) -> None: + """ + Args: + model_id: id of model to delete + Returns: + None + """ + storage_path = os.path.join(self._model_dir, f'{model_id}') + with self._lock_pool.get_lock(model_id).write_lock(): + if os.path.exists(storage_path): + for file_name in os.listdir(storage_path): + self._remove_from_cache(os.path.join(storage_path, file_name)) + shutil.rmtree(storage_path) + + def _remove_from_cache(self, file_path: str) -> None: + if file_path in self._model_cache: + del self._model_cache[file_path] diff --git a/iotdb-core/ainode/ainode/core/script.py b/iotdb-core/ainode/ainode/core/script.py new file mode 100644 index 0000000000000..b27bb6ab61bc0 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/script.py @@ -0,0 +1,177 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os +import shutil +import sys +from datetime import datetime + +import psutil + +from ainode.core.client import ClientManager +from ainode.core.config import AINodeDescriptor +from ainode.core.constant import TSStatusCode, AINODE_SYSTEM_FILE_NAME +from ainode.core.exception import MissingConfigError +from ainode.core.log import Logger +from ainode.core.service import RPCService +from ainode.thrift.common.ttypes import TAINodeLocation, TEndPoint, TAINodeConfiguration, TNodeResource +from ainode.thrift.confignode.ttypes import TNodeVersionInfo + +logger = Logger() + + +def _generate_configuration() -> TAINodeConfiguration: + location = TAINodeLocation(AINodeDescriptor().get_config().get_ainode_id(), + TEndPoint(AINodeDescriptor().get_config().get_ain_inference_rpc_address(), + AINodeDescriptor().get_config().get_ain_inference_rpc_port())) + resource = TNodeResource( + int(psutil.cpu_count()), + int(psutil.virtual_memory()[0]) + ) + + return TAINodeConfiguration(location, resource) + + +def _generate_version_info() -> TNodeVersionInfo: + return TNodeVersionInfo(AINodeDescriptor().get_config().get_version_info(), + AINodeDescriptor().get_config().get_build_info()) + + +def _check_path_permission(): + system_path = AINodeDescriptor().get_config().get_ain_system_dir() + if not os.path.exists(system_path): + try: + os.makedirs(system_path) + os.chmod(system_path, 0o777) + except PermissionError as e: + logger.error(e) + raise e + + +def start_ainode(): + _check_path_permission() + system_properties_file = os.path.join(AINodeDescriptor().get_config().get_ain_system_dir(), AINODE_SYSTEM_FILE_NAME) + if not os.path.exists(system_properties_file): + # If the system.properties file does not exist, the AINode will register to ConfigNode. + try: + logger.info('IoTDB-AINode is registering to ConfigNode...') + ainode_id = ClientManager().borrow_config_node_client().node_register( + AINodeDescriptor().get_config().get_cluster_name(), + _generate_configuration(), + _generate_version_info()) + AINodeDescriptor().get_config().set_ainode_id(ainode_id) + system_properties = { + 'ainode_id': ainode_id, + 'cluster_name': AINodeDescriptor().get_config().get_cluster_name(), + 'iotdb_version': AINodeDescriptor().get_config().get_version_info(), + 'commit_id': AINodeDescriptor().get_config().get_build_info(), + 'ain_rpc_address': AINodeDescriptor().get_config().get_ain_inference_rpc_address(), + 'ain_rpc_port': AINodeDescriptor().get_config().get_ain_inference_rpc_port(), + 'config_node_list': AINodeDescriptor().get_config().get_ain_target_config_node_list(), + } + with open(system_properties_file, 'w') as f: + f.write('#' + str(datetime.now()) + '\n') + for key, value in system_properties.items(): + f.write(key + '=' + str(value) + '\n') + + except Exception as e: + logger.error('IoTDB-AINode failed to register to ConfigNode: {}'.format(e)) + raise e + else: + # If the system.properties file does exist, the AINode will just restart. + try: + logger.info('IoTDB-AINode is restarting...') + ClientManager().borrow_config_node_client().node_restart( + AINodeDescriptor().get_config().get_cluster_name(), + _generate_configuration(), + _generate_version_info()) + + except Exception as e: + logger.error('IoTDB-AINode failed to restart: {}'.format(e)) + raise e + + rpc_service = RPCService() + rpc_service.start() + rpc_service.join(1) + if rpc_service.exit_code != 0: + return + + logger.info('IoTDB-AINode has successfully started.') + + +def remove_ainode(arguments): + # Delete the current node + if len(arguments) == 2: + target_ainode_id = AINodeDescriptor().get_config().get_ainode_id() + target_rpc_address = AINodeDescriptor().get_config().get_ain_inference_rpc_address() + target_rpc_port = AINodeDescriptor().get_config().get_ain_inference_rpc_port() + + # Delete the node with a given id + elif len(arguments) == 3: + target_ainode_id = int(arguments[2]) + ainode_configuration_map = ClientManager().borrow_config_node_client().get_ainode_configuration( + target_ainode_id) + + end_point = ainode_configuration_map[target_ainode_id].location.internalEndPoint + target_rpc_address = end_point.ip + target_rpc_port = end_point.port + + if not end_point: + raise MissingConfigError("NodeId: {} not found in cluster ".format(target_ainode_id)) + + logger.info('Got target AINode id: {}'.format(target_ainode_id)) + + else: + raise MissingConfigError("Invalid command") + + location = TAINodeLocation(target_ainode_id, TEndPoint(target_rpc_address, target_rpc_port)) + status = ClientManager().borrow_config_node_client().node_remove(location) + + if status.code == TSStatusCode.SUCCESS_STATUS.get_status_code(): + logger.info('IoTDB-AINode has successfully removed.') + if os.path.exists(AINodeDescriptor().get_config().get_ain_models_dir()): + shutil.rmtree(AINodeDescriptor().get_config().get_ain_models_dir()) + + +def main(): + arguments = sys.argv + # load config + AINodeDescriptor() + if len(arguments) == 1: + logger.info("Command line argument must be specified.") + return + command = arguments[1] + if command == 'start': + try: + logger.info('IoTDB-AINode is starting...') + start_ainode() + except Exception as e: + logger.error("Start AINode failed, because of: {}".format(e)) + sys.exit(1) + elif command == 'remove': + try: + logger.info("Removing AINode...") + remove_ainode(arguments) + except Exception as e: + logger.error("Remove AINode failed, because of: {}".format(e)) + sys.exit(1) + else: + logger.warning("Unknown argument: {}.".format(command)) + + +if __name__ == '__main__': + main() diff --git a/iotdb-core/ainode/ainode/core/service.py b/iotdb-core/ainode/ainode/core/service.py new file mode 100644 index 0000000000000..9532093d1dade --- /dev/null +++ b/iotdb-core/ainode/ainode/core/service.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import threading + +from thrift.protocol import TCompactProtocol, TBinaryProtocol +from thrift.server import TServer +from thrift.transport import TSocket, TTransport + +from ainode.core.config import AINodeDescriptor +from ainode.core.handler import AINodeRPCServiceHandler +from ainode.core.log import Logger +from ainode.thrift.ainode import IAINodeRPCService + +logger = Logger() + + +class RPCService(threading.Thread): + def __init__(self): + self.exit_code = 0 + super().__init__() + processor = IAINodeRPCService.Processor(handler=AINodeRPCServiceHandler()) + transport = TSocket.TServerSocket(host=AINodeDescriptor().get_config().get_ain_inference_rpc_address(), + port=AINodeDescriptor().get_config().get_ain_inference_rpc_port()) + transport_factory = TTransport.TFramedTransportFactory() + if AINodeDescriptor().get_config().get_ain_thrift_compression_enabled(): + protocol_factory = TCompactProtocol.TCompactProtocolFactory() + else: + protocol_factory = TBinaryProtocol.TBinaryProtocolFactory() + + self.__pool_server = TServer.TThreadPoolServer(processor, transport, transport_factory, protocol_factory) + + def run(self) -> None: + logger.info("The RPC service thread begin to run...") + try: + self.__pool_server.serve() + except Exception as e: + self.exit_code = 1 + logger.error(e) diff --git a/iotdb-core/ainode/ainode/core/util/__init__.py b/iotdb-core/ainode/ainode/core/util/__init__.py new file mode 100644 index 0000000000000..2a1e720805f29 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/iotdb-core/ainode/ainode/core/util/activation.py b/iotdb-core/ainode/ainode/core/util/activation.py new file mode 100644 index 0000000000000..25be5dc2b3996 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/activation.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import torch +import torch.nn as nn +import torch.nn.functional as F +from collections import OrderedDict + +class ClampedGELU(nn.Module): + def __init__(self, min_val=-10, max_val=10): + super().__init__() + self.act = nn.GELU() + self.min_val = min_val + self.max_val = max_val + + def forward(self, x): + return torch.clamp(self.act(x), self.min_val, self.max_val) + +class ClassInstantier(OrderedDict): + def __getitem__(self, key): + content = super().__getitem__(key) + cls, kwargs = content if isinstance(content, tuple) else (content, {}) + return cls(**kwargs) + +ACT2CLS = { + "gelu": nn.GELU, + "gelu_10": (ClampedGELU, {"min": -10, "max": 10}), + "leaky_relu": nn.LeakyReLU, + "relu": nn.ReLU, + "relu6": nn.ReLU6, + "sigmoid": nn.Sigmoid, + "silu": nn.SiLU, + "swish": nn.SiLU, + "tanh": nn.Tanh, + "prelu": nn.PReLU, +} +ACT2FN = ClassInstantier(ACT2CLS) \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/core/util/decorator.py b/iotdb-core/ainode/ainode/core/util/decorator.py new file mode 100644 index 0000000000000..33b9f4835ac8a --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/decorator.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +def singleton(cls): + instances = {} + + def get_instance(*args, **kwargs): + if cls not in instances: + instances[cls] = cls(*args, **kwargs) + return instances[cls] + + return get_instance diff --git a/iotdb-core/ainode/ainode/core/util/huggingface_cache.py b/iotdb-core/ainode/ainode/core/util/huggingface_cache.py new file mode 100644 index 0000000000000..1f8516f33defa --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/huggingface_cache.py @@ -0,0 +1,202 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +""" +copied from transformers.cache_utils.py(transformers==4.40.1) +""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple +import torch +import torch.nn as nn + +class Cache: + """ + Base, abstract class for all caches. The actual data structure is specific to each subclass. + """ + # def __init__(self): + # # to avoid torch.jit.script error + # super().__init__() + # self._seen_tokens = 0 + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. These are specific to each subclass and allow new types of + cache to be created. + + Return: + A tuple containing the updated key and value states. + """ + raise NotImplementedError("Make sure to implement `update` in a subclass.") + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + raise NotImplementedError("Make sure to implement `get_seq_length` in a subclass.") + + def get_max_length(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states, if there is any.""" + raise NotImplementedError("Make sure to implement `get_max_length` in a subclass.") + + def get_usable_length(self, new_seq_length: int, layer_idx: Optional[int] = 0) -> int: + """Given the sequence length of the new inputs, returns the usable length of the cache.""" + # Cache without size limit -> all cache is usable + # Cache with size limit -> if the length cache plus the length of the new inputs is larger the maximum cache + # length, we will need to evict part of the cache (and thus not all cache is usable) + max_length = self.get_max_length() + previous_seq_length = self.get_seq_length(layer_idx) + if max_length is not None and previous_seq_length + new_seq_length > max_length: + return max_length - new_seq_length + return previous_seq_length + + @property + def seen_tokens(self): + if hasattr(self, "_seen_tokens"): + return self._seen_tokens + else: + return None + +class DynamicCache(Cache): + """ + A cache that grows dynamically as more tokens are generated. This is the default for generative models. + + It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is + `[batch_size, num_heads, seq_len, head_dim]`. + """ + + def __init__(self) -> None: + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + self._seen_tokens = 0 # Used in `generate` to keep tally of how many tokens the cache has seen + + def __getitem__(self, layer_idx: int) -> List[Tuple[torch.Tensor]]: + """ + Support for backwards-compatible `past_key_value` indexing, e.g. `past_key_value[0][0].shape[2]` to get the + sequence length. + """ + if layer_idx < len(self): + return (self.key_cache[layer_idx], self.value_cache[layer_idx]) + else: + raise KeyError(f"Cache only has {len(self)} layers, attempted to access layer with index {layer_idx}") + + # def __iter__(self): + # """ + # Support for backwards-compatible `past_key_value` iteration, e.g. `for x in past_key_value:` to iterate over + # keys and values + # """ + # for layer_idx in range(len(self)): + # yield (self.key_cache[layer_idx], self.value_cache[layer_idx]) + + def __len__(self): + """ + Support for backwards-compatible `past_key_value` length, e.g. `len(past_key_value)`. This value corresponds + to the number of layers in the model. + """ + return len(self.key_cache) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. No additional arguments are used in `DynamicCache`. + + Return: + A tuple containing the updated key and value states. + """ + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + # Update the cache + if len(self.key_cache) <= layer_idx: + self.key_cache.append(key_states) + self.value_cache.append(value_states) + else: + self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + def get_seq_length(self, layer_idx: int = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + if len(self.key_cache) <= layer_idx: + return 0 + return self.key_cache[layer_idx].shape[-2] + + def get_max_length(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states. DynamicCache does not have a maximum length.""" + return None + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Reorders the cache for beam search, given the selected beam indices.""" + for layer_idx in range(len(self.key_cache)): + device = self.key_cache[layer_idx].device + self.key_cache[layer_idx] = self.key_cache[layer_idx].index_select(0, beam_idx.to(device)) + device = self.value_cache[layer_idx].device + self.value_cache[layer_idx] = self.value_cache[layer_idx].index_select(0, beam_idx.to(device)) + + def to_legacy_cache(self) -> Tuple[Tuple[torch.Tensor], Tuple[torch.Tensor]]: + """Converts the `DynamicCache` instance into the its equivalent in the legacy cache format.""" + legacy_cache = () + for layer_idx in range(len(self)): + legacy_cache += ((self.key_cache[layer_idx], self.value_cache[layer_idx]),) + return legacy_cache + + def init_data(self, past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None): + if past_key_values is not None: + for layer_idx in range(len(past_key_values)): + key_states, value_states = past_key_values[layer_idx] + self.update(key_states, value_states, layer_idx) + + @classmethod + def from_legacy_cache(cls, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None) -> "DynamicCache": + """Converts a cache in the legacy cache format into an equivalent `DynamicCache`.""" + cache = cls() + if past_key_values is not None: + for layer_idx in range(len(past_key_values)): + key_states, value_states = past_key_values[layer_idx] + cache.update(key_states, value_states, layer_idx) + return cache + \ No newline at end of file diff --git a/iotdb-core/ainode/ainode/core/util/lock.py b/iotdb-core/ainode/ainode/core/util/lock.py new file mode 100644 index 0000000000000..91abbedad249f --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/lock.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import hashlib +import threading + + +class ReadWriteLock: + def __init__(self): + self._reader_num = 0 + self._read_lock = threading.Lock() + self._write_lock = threading.Lock() + + def acquire_read(self): + with self._read_lock: + if self._reader_num == 0: + self._write_lock.acquire() + self._reader_num += 1 + + def release_read(self): + with self._read_lock: + self._reader_num -= 1 + if self._reader_num == 0: + self._write_lock.release() + + def acquire_write(self): + self._write_lock.acquire() + + def release_write(self): + self._write_lock.release() + + class ReadLockContext: + def __init__(self, rw_lock): + self.rw_lock = rw_lock + + def __enter__(self): + self.rw_lock.acquire_read() + + def __exit__(self, exc_type, exc_value, traceback): + self.rw_lock.release_read() + + class WriteLockContext: + def __init__(self, rw_lock): + self.rw_lock = rw_lock + + def __enter__(self): + self.rw_lock.acquire_write() + + def __exit__(self, exc_type, exc_value, traceback): + self.rw_lock.release_write() + + def read_lock(self): + return self.ReadLockContext(self) + + def write_lock(self): + return self.WriteLockContext(self) + + +def hash_model_id(model_id): + return int(hashlib.md5(str(model_id).encode()).hexdigest(), 16) + + +class ModelLockPool: + def __init__(self, pool_size=16): + self._pool = [ReadWriteLock() for _ in range(pool_size)] + self._pool_size = pool_size + + def get_lock(self, model_id): + pool_index = hash_model_id(model_id) % self._pool_size + return self._pool[pool_index] diff --git a/iotdb-core/ainode/ainode/core/util/masking.py b/iotdb-core/ainode/ainode/core/util/masking.py new file mode 100644 index 0000000000000..826e05d67cfb6 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/masking.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import torch + +class TriangularCausalMask(): + def __init__(self, B, L, device="cpu"): + mask_shape = [B, 1, L, L] + with torch.no_grad(): + self._mask = torch.triu(torch.ones(mask_shape, dtype=torch.bool), diagonal=1).to(device) + + @property + def mask(self): + return self._mask + +class TimerMultivariateMask(): + def __init__(self, B, n_vars, n_tokens, device="cpu"): + mask_shape = [B, 1, n_tokens, n_tokens] + with torch.no_grad(): + self._mask1 = torch.ones((n_vars, n_vars), dtype=torch.bool).to(device) + self._mask2 = torch.triu(torch.ones(mask_shape, dtype=torch.bool), diagonal=1).to(device) + self._mask = torch.kron(self._mask1, self._mask2) + @property + def mask(self): + return self._mask + +class TimerCovariateMask(): + def __init__(self, B, n_vars, n_tokens, device="cpu"): + mask_shape = [B, 1, n_tokens, n_tokens] + with torch.no_grad(): + self._mask1 = torch.eye(n_vars, dtype=torch.bool).to(device) + self._mask2 = torch.tril(torch.ones(mask_shape, dtype=torch.bool)).to(device) + self._mask = ~torch.kron(self._mask1, self._mask2) + self._mask[:, :, -n_tokens:, :-n_tokens] = False + + @property + def mask(self): + return self._mask + +def prepare_4d_causal_attention_mask( + attention_mask, + input_shape, # (B, T_query) + inputs_embeds: torch.Tensor, + past_key_values_length: int = 0, +): + B, T = input_shape + S = T + past_key_values_length + dtype, device = inputs_embeds.dtype, inputs_embeds.device + + # 1) causal mask + q_pos = torch.arange(past_key_values_length, + past_key_values_length + T, device=device) # [T] + k_pos = torch.arange(S, device=device) # [S] + causal = (k_pos.unsqueeze(0) <= q_pos.unsqueeze(1)) # [T,S] bool + + mask = torch.zeros((T, S), dtype=dtype, device=device) + mask.masked_fill_(~causal, torch.finfo(dtype).min) # unvisible → -inf + mask = mask.unsqueeze(0).unsqueeze(0) # [1,1,T,S] + + # 2) padding mask + if attention_mask is not None: + pad = (1.0 - attention_mask.to(dtype)) * torch.finfo(dtype).min # [B,S] + pad = pad[:, None, None, :] # [B,1,1,S] + else: + pad = 0. + + return mask + pad # [B,1,T,S] diff --git a/iotdb-core/ainode/ainode/core/util/serde.py b/iotdb-core/ainode/ainode/core/util/serde.py new file mode 100644 index 0000000000000..70b86d6609596 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/serde.py @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import struct +from enum import Enum + +import numpy as np +import pandas as pd + +from ainode.core.exception import BadConfigValueError + + +class TSDataType(Enum): + BOOLEAN = 0 + INT32 = 1 + INT64 = 2 + FLOAT = 3 + DOUBLE = 4 + TEXT = 5 + + # this method is implemented to avoid the issue reported by: + # https://bugs.python.org/issue30545 + def __eq__(self, other) -> bool: + return self.value == other.value + + def __hash__(self): + return self.value + + def np_dtype(self): + return { + TSDataType.BOOLEAN: np.dtype(">?"), + TSDataType.FLOAT: np.dtype(">f4"), + TSDataType.DOUBLE: np.dtype(">f8"), + TSDataType.INT32: np.dtype(">i4"), + TSDataType.INT64: np.dtype(">i8"), + TSDataType.TEXT: np.dtype("str"), + }[self] + + +TIMESTAMP_STR = "Time" +START_INDEX = 2 + + +# convert dataFrame to tsBlock in binary +# input shouldn't contain time column +def convert_to_binary(data_frame: pd.DataFrame): + data_shape = data_frame.shape + value_column_size = data_shape[1] + position_count = data_shape[0] + keys = data_frame.keys() + + binary = value_column_size.to_bytes(4, byteorder="big") + + for data_type in data_frame.dtypes: + binary += _get_type_in_byte(data_type) + + # position count + binary += position_count.to_bytes(4, byteorder="big") + + # column encoding + binary += b'\x02' + for data_type in data_frame.dtypes: + binary += _get_encoder(data_type) + + # write columns, the column in index 0 must be timeColumn + binary += bool.to_bytes(False, 1, byteorder="big") + for i in range(position_count): + value = 0 + v = struct.pack(">i", value) + binary += v + binary += v + + for i in range(value_column_size): + # the value can't be null + binary += bool.to_bytes(False, 1, byteorder="big") + col = data_frame[keys[i]] + for j in range(position_count): + value = col[j] + if value.dtype.byteorder != '>': + value = value.byteswap() + binary += value.tobytes() + + return binary + + +def _get_encoder(data_type: pd.Series): + if data_type == "bool": + return b'\x00' + elif data_type == "int32" or data_type == "float32": + return b'\x01' + elif data_type == "int64" or data_type == "float64": + return b'\x02' + elif data_type == "texr": + return b'\x03' + + +def _get_type_in_byte(data_type: pd.Series): + if data_type == 'bool': + return b'\x00' + elif data_type == 'int32': + return b'\x01' + elif data_type == 'int64': + return b'\x02' + elif data_type == 'float32': + return b'\x03' + elif data_type == 'float64': + return b'\x04' + elif data_type == 'text': + return b'\x05' + else: + raise BadConfigValueError('data_type', data_type, + "data_type should be in ['bool', 'int32', 'int64', 'float32', 'float64', 'text']") + + +# General Methods +def get_data_type_byte_from_str(value): + ''' + Args: + value (str): data type in ['bool', 'int32', 'int64', 'float32', 'float64', 'text'] + Returns: + byte: corresponding data type in [b'\x00', b'\x01', b'\x02', b'\x03', b'\x04', b'\x05'] + ''' + if value not in ['bool', 'int32', 'int64', 'float32', 'float64', 'text']: + raise BadConfigValueError('data_type', value, + "data_type should be in ['bool', 'int32', 'int64', 'float32', 'float64', 'text']") + if value == "bool": + return TSDataType.BOOLEAN.value + elif value == "int32": + return TSDataType.INT32.value + elif value == "int64": + return TSDataType.INT64.value + elif value == "float32": + return TSDataType.FLOAT.value + elif value == "float64": + return TSDataType.DOUBLE.value + elif value == "text": + return TSDataType.TEXT.value diff --git a/iotdb-core/ainode/ainode/core/util/status.py b/iotdb-core/ainode/ainode/core/util/status.py new file mode 100644 index 0000000000000..37368b0068b14 --- /dev/null +++ b/iotdb-core/ainode/ainode/core/util/status.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from ainode.core.constant import TSStatusCode +from ainode.core.log import Logger +from ainode.thrift.common.ttypes import TSStatus + + +def get_status(status_code: TSStatusCode, message: str = None) -> TSStatus: + status = TSStatus(status_code.get_status_code()) + status.message = message + return status + + +def verify_success(status: TSStatus, err_msg: str) -> None: + if status.code != TSStatusCode.SUCCESS_STATUS.get_status_code(): + Logger().warning(err_msg + ", error status is ", status) + raise RuntimeError(str(status.code) + ": " + status.message) diff --git a/iotdb-core/ainode/poetry.lock b/iotdb-core/ainode/poetry.lock new file mode 100644 index 0000000000000..bf82e69472327 --- /dev/null +++ b/iotdb-core/ainode/poetry.lock @@ -0,0 +1,1704 @@ +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. + +[[package]] +name = "alembic" +version = "1.13.2" +description = "A database migration tool for SQLAlchemy." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "alembic-1.13.2-py3-none-any.whl", hash = "sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953"}, + {file = "alembic-1.13.2.tar.gz", hash = "sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef"}, +] + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.9\""} +importlib-resources = {version = "*", markers = "python_version < \"3.9\""} +Mako = "*" +SQLAlchemy = ">=1.3.0" +typing-extensions = ">=4" + +[package.extras] +tz = ["backports.zoneinfo ; python_version < \"3.9\""] + +[[package]] +name = "certifi" +version = "2024.7.4" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +groups = ["main"] +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "colorlog" +version = "6.8.2" +description = "Add colours to the output of Python's logging module." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "colorlog-6.8.2-py3-none-any.whl", hash = "sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33"}, + {file = "colorlog-6.8.2.tar.gz", hash = "sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + +[[package]] +name = "cython" +version = "3.0.11" +description = "The Cython compiler for writing C extensions in the Python language." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] +files = [ + {file = "Cython-3.0.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:44292aae17524abb4b70a25111fe7dec1a0ad718711d47e3786a211d5408fdaa"}, + {file = "Cython-3.0.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75d45fbc20651c1b72e4111149fed3b33d270b0a4fb78328c54d965f28d55e1"}, + {file = "Cython-3.0.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d89a82937ce4037f092e9848a7bbcc65bc8e9fc9aef2bb74f5c15e7d21a73080"}, + {file = "Cython-3.0.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a8ea2e7e2d3bc0d8630dafe6c4a5a89485598ff8a61885b74f8ed882597efd5"}, + {file = "Cython-3.0.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cee29846471ce60226b18e931d8c1c66a158db94853e3e79bc2da9bd22345008"}, + {file = "Cython-3.0.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eeb6860b0f4bfa402de8929833fe5370fa34069c7ebacb2d543cb017f21fb891"}, + {file = "Cython-3.0.11-cp310-cp310-win32.whl", hash = "sha256:3699391125ab344d8d25438074d1097d9ba0fb674d0320599316cfe7cf5f002a"}, + {file = "Cython-3.0.11-cp310-cp310-win_amd64.whl", hash = "sha256:d02f4ebe15aac7cdacce1a628e556c1983f26d140fd2e0ac5e0a090e605a2d38"}, + {file = "Cython-3.0.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75ba1c70b6deeaffbac123856b8d35f253da13552207aa969078611c197377e4"}, + {file = "Cython-3.0.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af91497dc098718e634d6ec8f91b182aea6bb3690f333fc9a7777bc70abe8810"}, + {file = "Cython-3.0.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3999fb52d3328a6a5e8c63122b0a8bd110dfcdb98dda585a3def1426b991cba7"}, + {file = "Cython-3.0.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d566a4e09b8979be8ab9f843bac0dd216c81f5e5f45661a9b25cd162ed80508c"}, + {file = "Cython-3.0.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:46aec30f217bdf096175a1a639203d44ac73a36fe7fa3dd06bd012e8f39eca0f"}, + {file = "Cython-3.0.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ddd1fe25af330f4e003421636746a546474e4ccd8f239f55d2898d80983d20ed"}, + {file = "Cython-3.0.11-cp311-cp311-win32.whl", hash = "sha256:221de0b48bf387f209003508e602ce839a80463522fc6f583ad3c8d5c890d2c1"}, + {file = "Cython-3.0.11-cp311-cp311-win_amd64.whl", hash = "sha256:3ff8ac1f0ecd4f505db4ab051e58e4531f5d098b6ac03b91c3b902e8d10c67b3"}, + {file = "Cython-3.0.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:11996c40c32abf843ba652a6d53cb15944c88d91f91fc4e6f0028f5df8a8f8a1"}, + {file = "Cython-3.0.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63f2c892e9f9c1698ecfee78205541623eb31cd3a1b682668be7ac12de94aa8e"}, + {file = "Cython-3.0.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b14c24f1dc4c4c9d997cca8d1b7fb01187a218aab932328247dcf5694a10102"}, + {file = "Cython-3.0.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8eed5c015685106db15dd103fd040948ddca9197b1dd02222711815ea782a27"}, + {file = "Cython-3.0.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780f89c95b8aec1e403005b3bf2f0a2afa060b3eba168c86830f079339adad89"}, + {file = "Cython-3.0.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a690f2ff460682ea985e8d38ec541be97e0977fa0544aadc21efc116ff8d7579"}, + {file = "Cython-3.0.11-cp312-cp312-win32.whl", hash = "sha256:2252b5aa57621848e310fe7fa6f7dce5f73aa452884a183d201a8bcebfa05a00"}, + {file = "Cython-3.0.11-cp312-cp312-win_amd64.whl", hash = "sha256:da394654c6da15c1d37f0b7ec5afd325c69a15ceafee2afba14b67a5df8a82c8"}, + {file = "Cython-3.0.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4341d6a64d47112884e0bcf31e6c075268220ee4cd02223047182d4dda94d637"}, + {file = "Cython-3.0.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:351955559b37e6c98b48aecb178894c311be9d731b297782f2b78d111f0c9015"}, + {file = "Cython-3.0.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c02361af9bfa10ff1ccf967fc75159e56b1c8093caf565739ed77a559c1f29f"}, + {file = "Cython-3.0.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6823aef13669a32caf18bbb036de56065c485d9f558551a9b55061acf9c4c27f"}, + {file = "Cython-3.0.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6fb68cef33684f8cc97987bee6ae919eee7e18ee6a3ad7ed9516b8386ef95ae6"}, + {file = "Cython-3.0.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:790263b74432cb997740d73665f4d8d00b9cd1cecbdd981d93591ddf993d4f12"}, + {file = "Cython-3.0.11-cp313-cp313-win32.whl", hash = "sha256:e6dd395d1a704e34a9fac00b25f0036dce6654c6b898be6f872ac2bb4f2eda48"}, + {file = "Cython-3.0.11-cp313-cp313-win_amd64.whl", hash = "sha256:52186101d51497519e99b60d955fd5cb3bf747c67f00d742e70ab913f1e42d31"}, + {file = "Cython-3.0.11-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c69d5cad51388522b98a99b4be1b77316de85b0c0523fa865e0ea58bbb622e0a"}, + {file = "Cython-3.0.11-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8acdc87e9009110adbceb7569765eb0980129055cc954c62f99fe9f094c9505e"}, + {file = "Cython-3.0.11-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dd47865f4c0a224da73acf83d113f93488d17624e2457dce1753acdfb1cc40c"}, + {file = "Cython-3.0.11-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:301bde949b4f312a1c70e214b0c3bc51a3f955d466010d2f68eb042df36447b0"}, + {file = "Cython-3.0.11-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:f3953d2f504176f929862e5579cfc421860c33e9707f585d70d24e1096accdf7"}, + {file = "Cython-3.0.11-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:3f2b062f6df67e8a56c75e500ca330cf62c85ac26dd7fd006f07ef0f83aebfa3"}, + {file = "Cython-3.0.11-cp36-cp36m-win32.whl", hash = "sha256:c3d68751668c66c7a140b6023dba5d5d507f72063407bb609d3a5b0f3b8dfbe4"}, + {file = "Cython-3.0.11-cp36-cp36m-win_amd64.whl", hash = "sha256:bcd29945fafd12484cf37b1d84f12f0e7a33ba3eac5836531c6bd5283a6b3a0c"}, + {file = "Cython-3.0.11-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4e9a8d92978b15a0c7ca7f98447c6c578dc8923a0941d9d172d0b077cb69c576"}, + {file = "Cython-3.0.11-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:421017466e9260aca86823974e26e158e6358622f27c0f4da9c682f3b6d2e624"}, + {file = "Cython-3.0.11-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d80a7232938d523c1a12f6b1794ab5efb1ae77ad3fde79de4bb558d8ab261619"}, + {file = "Cython-3.0.11-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfa550d9ae39e827a6e7198076df763571cb53397084974a6948af558355e028"}, + {file = "Cython-3.0.11-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:aedceb6090a60854b31bf9571dc55f642a3fa5b91f11b62bcef167c52cac93d8"}, + {file = "Cython-3.0.11-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:473d35681d9f93ce380e6a7c8feb2d65fc6333bd7117fbc62989e404e241dbb0"}, + {file = "Cython-3.0.11-cp37-cp37m-win32.whl", hash = "sha256:3379c6521e25aa6cd7703bb7d635eaca75c0f9c7f1b0fdd6dd15a03bfac5f68d"}, + {file = "Cython-3.0.11-cp37-cp37m-win_amd64.whl", hash = "sha256:14701edb3107a5d9305a82d9d646c4f28bfecbba74b26cc1ee2f4be08f602057"}, + {file = "Cython-3.0.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:598699165cfa7c6d69513ee1bffc9e1fdd63b00b624409174c388538aa217975"}, + {file = "Cython-3.0.11-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0583076c4152b417a3a8a5d81ec02f58c09b67d3f22d5857e64c8734ceada8c"}, + {file = "Cython-3.0.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52205347e916dd65d2400b977df4c697390c3aae0e96275a438cc4ae85dadc08"}, + {file = "Cython-3.0.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:989899a85f0d9a57cebb508bd1f194cb52f0e3f7e22ac259f33d148d6422375c"}, + {file = "Cython-3.0.11-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:53b6072a89049a991d07f42060f65398448365c59c9cb515c5925b9bdc9d71f8"}, + {file = "Cython-3.0.11-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:f988f7f8164a6079c705c39e2d75dbe9967e3dacafe041420d9af7b9ee424162"}, + {file = "Cython-3.0.11-cp38-cp38-win32.whl", hash = "sha256:a1f4cbc70f6b7f0c939522118820e708e0d490edca42d852fa8004ec16780be2"}, + {file = "Cython-3.0.11-cp38-cp38-win_amd64.whl", hash = "sha256:187685e25e037320cae513b8cc4bf9dbc4465c037051aede509cbbf207524de2"}, + {file = "Cython-3.0.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0fc6fdd6fa493be7bdda22355689d5446ac944cd71286f6f44a14b0d67ee3ff5"}, + {file = "Cython-3.0.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b1d1f6f94cc5d42a4591f6d60d616786b9cd15576b112bc92a23131fcf38020"}, + {file = "Cython-3.0.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4ab2b92a3e6ed552adbe9350fd2ef3aa0cc7853cf91569f9dbed0c0699bbeab"}, + {file = "Cython-3.0.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:104d6f2f2c827ccc5e9e42c80ef6773a6aa94752fe6bc5b24a4eab4306fb7f07"}, + {file = "Cython-3.0.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:13062ce556a1e98d2821f7a0253b50569fdc98c36efd6653a65b21e3f8bbbf5f"}, + {file = "Cython-3.0.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:525d09b3405534763fa73bd78c8e51ac8264036ce4c16d37dfd1555a7da6d3a7"}, + {file = "Cython-3.0.11-cp39-cp39-win32.whl", hash = "sha256:b8c7e514075696ca0f60c337f9e416e61d7ccbc1aa879a56c39181ed90ec3059"}, + {file = "Cython-3.0.11-cp39-cp39-win_amd64.whl", hash = "sha256:8948802e1f5677a673ea5d22a1e7e273ca5f83e7a452786ca286eebf97cee67c"}, + {file = "Cython-3.0.11-py2.py3-none-any.whl", hash = "sha256:0e25f6425ad4a700d7f77cd468da9161e63658837d1bc34861a9861a4ef6346d"}, + {file = "cython-3.0.11.tar.gz", hash = "sha256:7146dd2af8682b4ca61331851e6aebce9fe5158e75300343f80c07ca80b1faff"}, +] + +[[package]] +name = "dynaconf" +version = "3.2.6" +description = "The dynamic configurator for your Python Project" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "dynaconf-3.2.6-py2.py3-none-any.whl", hash = "sha256:3911c740d717df4576ed55f616c7cbad6e06bc8ef23ffca444b6e2a12fb1c34c"}, + {file = "dynaconf-3.2.6.tar.gz", hash = "sha256:74cc1897396380bb957730eb341cc0976ee9c38bbcb53d3307c50caed0aedfb8"}, +] + +[package.extras] +all = ["configobj", "hvac", "redis", "ruamel.yaml"] +configobj = ["configobj"] +ini = ["configobj"] +redis = ["redis"] +test = ["configobj", "django", "flask (>=0.12)", "hvac (>=1.1.0)", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist", "python-dotenv", "radon", "redis", "toml"] +toml = ["toml"] +vault = ["hvac"] +yaml = ["ruamel.yaml"] + +[[package]] +name = "filelock" +version = "3.15.4" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] +typing = ["typing-extensions (>=4.8) ; python_version < \"3.11\""] + +[[package]] +name = "fsspec" +version = "2024.6.1" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"}, + {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dev = ["pre-commit", "ruff"] +doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] +test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +tqdm = ["tqdm"] + +[[package]] +name = "greenlet" +version = "3.0.3" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + +[[package]] +name = "hmmlearn" +version = "0.3.2" +description = "Hidden Markov Models in Python with scikit-learn like API" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "hmmlearn-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:428eb8a60d5455bacf391d54abff876931ea5cf234cd10e0a4a921546652b466"}, + {file = "hmmlearn-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1e54a15b3fb47e1ba72c3e40dce9368b3aa2f3f8d4d94ed6684b999cd9e1528"}, + {file = "hmmlearn-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19124c22b7579d76b07be8c2128153de9071281bff7a0bc29cc3bd83baf0c4cf"}, + {file = "hmmlearn-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:011b1aaa67551151b9442670b58fa692e135ea6fd5dc76022e595964594a3654"}, + {file = "hmmlearn-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:3c4d643a40c21732df9eea70d1a0ed256a6aa28ed631762c062c6cce2ee6cb47"}, + {file = "hmmlearn-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:02df950f8ee4de9226bef55c1a7a947f706720990bf38e5e6da84b7418af9095"}, + {file = "hmmlearn-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ced6acc12ffadb3d2b03fdf0e52bc8211977202db170f5b0f62014199cffc224"}, + {file = "hmmlearn-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16d6b05065963155a0294b0ef3643f01a1897c0e00ab258e5155537e05027a2f"}, + {file = "hmmlearn-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16535843dfa38f619355f4f082d1e37622198a8df315c932f406d31f5786821b"}, + {file = "hmmlearn-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:c8c2511fb5c7ed31c98fe46b150991c20abb5d95e3c450bb449adfe792f4f9ee"}, + {file = "hmmlearn-0.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:29106ab096c1bd9a8dba8de5e6a8da763d899e4294de5db4a47be2fd28c73a41"}, + {file = "hmmlearn-0.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5ffe17028c881b0213803db604a7051384179877ee74ded76c02f797ccfe2c34"}, + {file = "hmmlearn-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25572410c59f4d278b56c1bdc8196f9efe4672b4bf135e3593c6e879cea94518"}, + {file = "hmmlearn-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b30a8b380916ccc299984594a55829131d2da1d66bbd6861bfc548e5d9b6984"}, + {file = "hmmlearn-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:b042bbb90488ccaaf5616b2ecd7687b17d733016df68192ad6f2a8d58bb96291"}, + {file = "hmmlearn-0.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:42308b50c2eb68ab541573c196c1123c64a50d0dba2b30b1e21e355084041738"}, + {file = "hmmlearn-0.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:47ef458bbae23f1b80bf923eca8a838778e550aa9397a39655aa97304fbeba69"}, + {file = "hmmlearn-0.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8a92724d2c7b7f8c2b8f54c8bfe061b75dc018d81041172ce435d24456232ed"}, + {file = "hmmlearn-0.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e99980c9a92ed4fd62307d6a91e3ff136ee7233f9f02a62038b68473741204e1"}, + {file = "hmmlearn-0.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:4375627550c0282587ec404cb1e2b59d5bff70522bfac3dc57f1a2912e4af9c4"}, + {file = "hmmlearn-0.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:81cc7f0cf33d8d114423347f00d45cd2fba926a66dd563e97bb0c9b3790e3ae2"}, + {file = "hmmlearn-0.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9c55c09ef8dee1deae31a51de9a99ceea0f86cfeb5eba879b1b6c7722d29eb95"}, + {file = "hmmlearn-0.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722a0f116c11160c97362f343d7ba0a78b03dd647cb0e2240127520d5d8d1580"}, + {file = "hmmlearn-0.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01b10877f21f6faab8b01ff63e92e93f614097d03a9fb7fc27a26a16227cd980"}, + {file = "hmmlearn-0.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:160c0018f71b6cbe79644ea743888c6e2d1bec6b09bec5e9b389e1796b57eec5"}, + {file = "hmmlearn-0.3.2.tar.gz", hash = "sha256:edaf485fdb1ea88da9ac642b2006c63d9950dd15d4d132f7205305d383e6f745"}, +] + +[package.dependencies] +numpy = ">=1.10" +scikit-learn = ">=0.16,<0.22.0 || >0.22.0" +scipy = ">=0.19" + +[package.extras] +docs = ["matplotlib", "pydata-sphinx-theme", "sphinx (>=2.0)", "sphinx-gallery"] +tests = ["pytest"] + +[[package]] +name = "idna" +version = "3.8" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, +] + +[[package]] +name = "importlib-metadata" +version = "8.4.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.9\"" +files = [ + {file = "importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1"}, + {file = "importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +test = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] + +[[package]] +name = "importlib-resources" +version = "6.4.4" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.9\"" +files = [ + {file = "importlib_resources-6.4.4-py3-none-any.whl", hash = "sha256:dda242603d1c9cd836c3368b1174ed74cb4049ecd209e7a1a0104620c18c5c11"}, + {file = "importlib_resources-6.4.4.tar.gz", hash = "sha256:20600c8b7361938dc0bb2d5ec0297802e575df486f5a544fa414da65e13721f7"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy"] + +[[package]] +name = "jinja2" +version = "3.1.6" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "joblib" +version = "1.4.2" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, +] + +[[package]] +name = "mako" +version = "1.3.5" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "Mako-1.3.5-py3-none-any.whl", hash = "sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a"}, + {file = "Mako-1.3.5.tar.gz", hash = "sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc"}, +] + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["Babel"] +lingua = ["lingua"] +testing = ["pytest"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] +tests = ["pytest (>=4.6)"] + +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + +[[package]] +name = "numpy" +version = "1.24.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, + {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, + {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, + {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, + {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, + {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, + {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, + {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, + {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, + {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.1.3.1" +description = "CUBLAS native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:ee53ccca76a6fc08fb9701aa95b6ceb242cdaab118c3bb152af4e579af792728"}, + {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-win_amd64.whl", hash = "sha256:2b964d60e8cf11b5e1073d179d85fa340c120e99b3067558f3cf98dd69d02906"}, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.1.105" +description = "CUDA profiling tools runtime libs." +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:e54fde3983165c624cb79254ae9818a456eb6e87a7fd4d56a2352c24ee542d7e"}, + {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:bea8236d13a0ac7190bd2919c3e8e6ce1e402104276e6f9694479e48bb0eb2a4"}, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.1.105" +description = "NVRTC native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:339b385f50c309763ca65456ec75e17bbefcbbf2893f462cb8b90584cd27a1c2"}, + {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:0a98a522d9ff138b96c010a65e145dc1b4850e9ecb75a0172371793752fd46ed"}, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.1.105" +description = "CUDA Runtime native Libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:6e258468ddf5796e25f1dc591a31029fa317d97a0a94ed93468fc86301d61e40"}, + {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:dfb46ef84d73fababab44cf03e3b83f80700d27ca300e537f85f636fac474344"}, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "8.9.2.26" +description = "cuDNN runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl", hash = "sha256:5ccb288774fdfb07a7e7025ffec286971c06d8d7b4fb162525334616d7629ff9"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.0.2.54" +description = "CUFFT native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl", hash = "sha256:794e3948a1aa71fd817c3775866943936774d1c14e7628c74f6f7417224cdf56"}, + {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-win_amd64.whl", hash = "sha256:d9ac353f78ff89951da4af698f80870b1534ed69993f10a4cf1d96f21357e253"}, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.2.106" +description = "CURAND native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:9d264c5036dde4e64f1de8c50ae753237c12e0b1348738169cd0f8a536c0e1e0"}, + {file = "nvidia_curand_cu12-10.3.2.106-py3-none-win_amd64.whl", hash = "sha256:75b6b0c574c0037839121317e17fd01f8a69fd2ef8e25853d826fec30bdba74a"}, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.4.5.107" +description = "CUDA solver native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl", hash = "sha256:8a7ec542f0412294b15072fa7dab71d31334014a69f953004ea7a118206fe0dd"}, + {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-win_amd64.whl", hash = "sha256:74e0c3a24c78612192a74fcd90dd117f1cf21dea4822e66d89e8ea80e3cd2da5"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" +nvidia-cusparse-cu12 = "*" +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.1.0.106" +description = "CUSPARSE native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:f3b50f42cf363f86ab21f720998517a659a48131e8d538dc02f8768237bd884c"}, + {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-win_amd64.whl", hash = "sha256:b798237e81b9719373e8fae8d4f091b70a0cf09d9d85c95a557e11df2d8e9a5a"}, +] + +[package.dependencies] +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.19.3" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl", hash = "sha256:a9734707a2c96443331c1e48c717024aa6678a0e2a4cb66b2c364d18cee6b48d"}, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.6.68" +description = "Nvidia JIT LTO Library" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b3fd0779845f68b92063ab1393abab1ed0a23412fc520df79a8190d098b5cd6b"}, + {file = "nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_x86_64.whl", hash = "sha256:125a6c2a44e96386dda634e13d944e60b07a0402d391a070e8fb4104b34ea1ab"}, + {file = "nvidia_nvjitlink_cu12-12.6.68-py3-none-win_amd64.whl", hash = "sha256:a55744c98d70317c5e23db14866a8cc2b733f7324509e941fc96276f9f37801d"}, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.1.105" +description = "NVIDIA Tools Extension" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:dc21cf308ca5691e7c04d962e213f8a4aa9bbfa23d95412f452254c2caeb09e5"}, + {file = "nvidia_nvtx_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:65f4d98982b31b60026e0e6de73fbdfc09d08a96f4656dd3665ca616a11e1e82"}, +] + +[[package]] +name = "optuna" +version = "3.6.1" +description = "A hyperparameter optimization framework" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "optuna-3.6.1-py3-none-any.whl", hash = "sha256:b32e0490bd6552790b70ec94de77dd2855057c9e229cd9f4da48fe8a31c7f1cc"}, + {file = "optuna-3.6.1.tar.gz", hash = "sha256:146e530b57b4b9afd7526b3e642fbe65491f7e292b405913355f8e438e361ecf"}, +] + +[package.dependencies] +alembic = ">=1.5.0" +colorlog = "*" +numpy = "*" +packaging = ">=20.0" +PyYAML = "*" +sqlalchemy = ">=1.3.0" +tqdm = "*" + +[package.extras] +benchmark = ["asv (>=0.5.0)", "botorch", "cma", "virtualenv"] +checking = ["black", "blackdoc", "flake8", "isort", "mypy", "mypy-boto3-s3", "types-PyYAML", "types-redis", "types-setuptools", "types-tqdm", "typing-extensions (>=3.10.0.0)"] +document = ["ase", "cmaes (>=0.10.0)", "fvcore", "lightgbm", "matplotlib (!=3.6.0)", "pandas", "pillow", "plotly (>=4.9.0)", "scikit-learn", "sphinx", "sphinx-copybutton", "sphinx-gallery", "sphinx-plotly-directive", "sphinx-rtd-theme (>=1.2.0)", "torch", "torchvision"] +optional = ["boto3", "cmaes (>=0.10.0)", "google-cloud-storage", "matplotlib (!=3.6.0)", "pandas", "plotly (>=4.9.0)", "redis", "scikit-learn (>=0.24.2)", "scipy", "torch"] +test = ["coverage", "fakeredis[lua]", "kaleido", "moto", "pytest", "scipy (>=1.9.2) ; python_version >= \"3.8\"", "torch"] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pandas" +version = "1.5.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version == \"3.10\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "patsy" +version = "0.5.6" +description = "A Python package for describing statistical models and for building design matrices." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "patsy-0.5.6-py2.py3-none-any.whl", hash = "sha256:19056886fd8fa71863fa32f0eb090267f21fb74be00f19f5c70b2e9d76c883c6"}, + {file = "patsy-0.5.6.tar.gz", hash = "sha256:95c6d47a7222535f84bff7f63d7303f2e297747a598db89cf5c67f0c0c7d2cdb"}, +] + +[package.dependencies] +numpy = ">=1.4" +six = "*" + +[package.extras] +test = ["pytest", "pytest-cov", "scipy"] + +[[package]] +name = "pmdarima" +version = "2.0.4" +description = "Python's forecast::auto.arima equivalent" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "pmdarima-2.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8e6c16672e0f122e63ab1d7282a362c762783264a07636bbc9d4ae3d58ac7605"}, + {file = "pmdarima-2.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c9a839a477100331c47aa8a841c198ecb0b3fab4aff958622d31fec86d2aea76"}, + {file = "pmdarima-2.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccd9e2186ba1ef45006f6c88dc9ecd6121ddb8914114bebcfa0d42899b40ced7"}, + {file = "pmdarima-2.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:165bdf787f5dafd5faab543d20524413b765d9cb6f020f0e1846551e7678414a"}, + {file = "pmdarima-2.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:cbde33812c37f441ba70d9e7b0479c758d56ad77a8dcbdead1fb8baad9aee806"}, + {file = "pmdarima-2.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d961c99b445e53eadf6627c61bf800f403bd247b7ec2f62c6dfe8a0b1bcbf0b"}, + {file = "pmdarima-2.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:78e815c51074411bbe8433a912af4cc8dac394d9330cfedf57dd5ce08efe4a65"}, + {file = "pmdarima-2.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8022484256492bc425f0053b3a0346ab0a877f0f668664be738fe07f6b423c08"}, + {file = "pmdarima-2.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46677b68efffde66aa1291799b39b91420961967fa2b6e041e26bb263782d38d"}, + {file = "pmdarima-2.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:bc594dd981bca5217b4448c96e82dbd60553b07263517a5cb0510b4bfe66ded1"}, + {file = "pmdarima-2.0.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:62a4ff308fbb5074a66c0877ba6b472d0fc406cbc0b5a2dba7e8fa800c9dd8ca"}, + {file = "pmdarima-2.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7e215ec6130b917843d26e0698637f976e4a6cc9dbd521bf44547668d08f058a"}, + {file = "pmdarima-2.0.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d837bc00266a234d9292914f19a67e04ca8361d9c309b70fc8f16c1ed863551"}, + {file = "pmdarima-2.0.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8790bff665a5ebaa36ddbbfd0e8ea51ad2e5809270dc74d462d5c2498b26220f"}, + {file = "pmdarima-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:a8bf7913bdbd0e286489b2111080a0f51f5d9d3ee5e05b7011a691207047ddcf"}, + {file = "pmdarima-2.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5bce63dca165948a2311eff0c200e57370829855fce6e7d3fe930497f2f9fc04"}, + {file = "pmdarima-2.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c1ed01cbb399d9cdbe3f12f2ef505a144826769ec3ad75f6075cadbe8447a13"}, + {file = "pmdarima-2.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73060d28ffabeae7374ab48ba7882b29ae6998de3e3e34f5484c64e0baefda0d"}, + {file = "pmdarima-2.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:776e21e076393c6fe86895ffe71e6769c9b3fe0dffb92d6f6558657580cf0a42"}, + {file = "pmdarima-2.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec865e8fb07378c470f3e41a77d6705875a674cefbe6c0185e9bc070e642da5c"}, + {file = "pmdarima-2.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:670ca1f93ed4f8239e7fbd7d1dd156108899e15e8fb0717b2b3fa605fa6ace35"}, + {file = "pmdarima-2.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab1ee166f511d2497d6b357bf0cac84326efff25349eb777aea5b030ed6bf8bb"}, + {file = "pmdarima-2.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9ead43510adfe3c0d4000e37427bfcf11ba6cc3b26091368a847c5da010e052"}, + {file = "pmdarima-2.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:c1213f3fa1e3ced9796f4092f9bd4be1881205f77bc2f5a1494695134a92000e"}, + {file = "pmdarima-2.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e0e3c90e7a91b44599f08cd9c62880466860e76bd1b0ca2d2ff8e72834a1a7f"}, + {file = "pmdarima-2.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8fe612384e4989f010dacdefbff874231f5b7351dfb84bcbfe9ed8d718fc4115"}, + {file = "pmdarima-2.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db842d97905b171c867aae1a492b8c958ec1bae987c3ee41c561a06d99a19efd"}, + {file = "pmdarima-2.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a03936d681c720b0e44565d099de2619b762cb9d443f3043de9a78888aff6bb2"}, + {file = "pmdarima-2.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:13ba7061d4e9d48f21e1c393bfaa3d6b31f60a8c97ddbe8d455fa675c594f9e4"}, + {file = "pmdarima-2.0.4.tar.gz", hash = "sha256:b87f9d9f5b7dc2ddbd053687c2264e26ac98fd4118e843c7e9bc3dd7343e5c1a"}, +] + +[package.dependencies] +Cython = ">=0.29,<0.29.18 || >0.29.18,<0.29.31 || >0.29.31" +joblib = ">=0.11" +numpy = ">=1.21.2" +packaging = ">=17.1" +pandas = ">=0.19" +scikit-learn = ">=0.22" +scipy = ">=1.3.2" +setuptools = ">=38.6.0,<50.0.0 || >50.0.0" +statsmodels = ">=0.13.2" +urllib3 = "*" + +[[package]] +name = "psutil" +version = "5.9.8" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +groups = ["main"] +files = [ + {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"}, + {file = "psutil-5.9.8-cp27-none-win32.whl", hash = "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"}, + {file = "psutil-5.9.8-cp27-none-win_amd64.whl", hash = "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"}, + {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"}, + {file = "psutil-5.9.8-cp36-cp36m-win32.whl", hash = "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"}, + {file = "psutil-5.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"}, + {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"}, + {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"}, + {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"}, + {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"}, +] + +[package.extras] +test = ["enum34 ; python_version <= \"3.4\"", "ipaddress ; python_version < \"3.0\"", "mock ; python_version < \"3.0\"", "pywin32 ; sys_platform == \"win32\"", "wmi ; sys_platform == \"win32\""] + +[[package]] +name = "pylru" +version = "1.2.1" +description = "A least recently used (LRU) cache implementation" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pylru-1.2.1-py3-none-any.whl", hash = "sha256:b7c75b0676e2fbae647823bc209e23998772867d3679f1583c7350a9b02a59f0"}, + {file = "pylru-1.2.1.tar.gz", hash = "sha256:47ad140a63ab9389648dadfbb4330700e0ffeeb28ec04664ee47d37ed133b0f4"}, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "scikit-base" +version = "0.6.2" +description = "Base classes for sklearn-like parametric objects" +optional = false +python-versions = "<3.13,>=3.8" +groups = ["main"] +files = [ + {file = "scikit-base-0.6.2.tar.gz", hash = "sha256:ac7c1dd9b1006e1e466d8269074f7fb02b7f5143c615f7fdf6a1e0c7565431fe"}, + {file = "scikit_base-0.6.2-py3-none-any.whl", hash = "sha256:91af9fdc6f8e35cbb1a89436f127681a64957645be0039a38d343381abef9154"}, +] + +[package.extras] +all-extras = ["numpy", "pandas"] +binder = ["jupyter"] +dev = ["pre-commit", "pytest", "pytest-cov", "scikit-learn (>=0.24.0)"] +docs = ["Sphinx (!=7.2.0,<8.0.0)", "jupyter", "myst-parser", "nbsphinx (>=0.8.6)", "numpydoc", "pydata-sphinx-theme", "sphinx-design (<0.6.0)", "sphinx-gallery (<0.16.0)", "sphinx-issues (<4.0.0)", "sphinx-panels", "tabulate"] +linters = ["black", "doc8", "flake8", "flake8-bugbear", "flake8-builtins", "flake8-comprehensions", "flake8-print", "flake8-quotes", "isort", "mypy", "nbqa", "pandas-vet", "pep8-naming", "pydocstyle"] +test = ["coverage", "numpy", "pandas", "pytest", "pytest-cov", "safety", "scikit-learn (>=0.24.0)", "scipy"] + +[[package]] +name = "scikit-learn" +version = "1.3.2" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "scikit-learn-1.3.2.tar.gz", hash = "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05"}, + {file = "scikit_learn-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1"}, + {file = "scikit_learn-1.3.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:535805c2a01ccb40ca4ab7d081d771aea67e535153e35a1fd99418fcedd1648a"}, + {file = "scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1215e5e58e9880b554b01187b8c9390bf4dc4692eedeaf542d3273f4785e342c"}, + {file = "scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ee107923a623b9f517754ea2f69ea3b62fc898a3641766cb7deb2f2ce450161"}, + {file = "scikit_learn-1.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:35a22e8015048c628ad099da9df5ab3004cdbf81edc75b396fd0cff8699ac58c"}, + {file = "scikit_learn-1.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6fb6bc98f234fda43163ddbe36df8bcde1d13ee176c6dc9b92bb7d3fc842eb66"}, + {file = "scikit_learn-1.3.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:18424efee518a1cde7b0b53a422cde2f6625197de6af36da0b57ec502f126157"}, + {file = "scikit_learn-1.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3271552a5eb16f208a6f7f617b8cc6d1f137b52c8a1ef8edf547db0259b2c9fb"}, + {file = "scikit_learn-1.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4144a5004a676d5022b798d9e573b05139e77f271253a4703eed295bde0433"}, + {file = "scikit_learn-1.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:67f37d708f042a9b8d59551cf94d30431e01374e00dc2645fa186059c6c5d78b"}, + {file = "scikit_learn-1.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8db94cd8a2e038b37a80a04df8783e09caac77cbe052146432e67800e430c028"}, + {file = "scikit_learn-1.3.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:61a6efd384258789aa89415a410dcdb39a50e19d3d8410bd29be365bcdd512d5"}, + {file = "scikit_learn-1.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb06f8dce3f5ddc5dee1715a9b9f19f20d295bed8e3cd4fa51e1d050347de525"}, + {file = "scikit_learn-1.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b2de18d86f630d68fe1f87af690d451388bb186480afc719e5f770590c2ef6c"}, + {file = "scikit_learn-1.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:0402638c9a7c219ee52c94cbebc8fcb5eb9fe9c773717965c1f4185588ad3107"}, + {file = "scikit_learn-1.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a19f90f95ba93c1a7f7924906d0576a84da7f3b2282ac3bfb7a08a32801add93"}, + {file = "scikit_learn-1.3.2-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:b8692e395a03a60cd927125eef3a8e3424d86dde9b2370d544f0ea35f78a8073"}, + {file = "scikit_learn-1.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15e1e94cc23d04d39da797ee34236ce2375ddea158b10bee3c343647d615581d"}, + {file = "scikit_learn-1.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:785a2213086b7b1abf037aeadbbd6d67159feb3e30263434139c98425e3dcfcf"}, + {file = "scikit_learn-1.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:64381066f8aa63c2710e6b56edc9f0894cc7bf59bd71b8ce5613a4559b6145e0"}, + {file = "scikit_learn-1.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6c43290337f7a4b969d207e620658372ba3c1ffb611f8bc2b6f031dc5c6d1d03"}, + {file = "scikit_learn-1.3.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:dc9002fc200bed597d5d34e90c752b74df516d592db162f756cc52836b38fe0e"}, + {file = "scikit_learn-1.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d08ada33e955c54355d909b9c06a4789a729977f165b8bae6f225ff0a60ec4a"}, + {file = "scikit_learn-1.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763f0ae4b79b0ff9cca0bf3716bcc9915bdacff3cebea15ec79652d1cc4fa5c9"}, + {file = "scikit_learn-1.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:ed932ea780517b00dae7431e031faae6b49b20eb6950918eb83bd043237950e0"}, +] + +[package.dependencies] +joblib = ">=1.1.1" +numpy = ">=1.17.3,<2.0" +scipy = ">=1.5.0" +threadpoolctl = ">=2.0.0" + +[package.extras] +benchmark = ["matplotlib (>=3.1.3)", "memory-profiler (>=0.57.0)", "pandas (>=1.0.5)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.1.3)", "memory-profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)", "sphinx (>=6.0.0)", "sphinx-copybutton (>=0.5.2)", "sphinx-gallery (>=0.10.1)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] +examples = ["matplotlib (>=3.1.3)", "pandas (>=1.0.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)"] +tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.16.2)"] + +[[package]] +name = "scipy" +version = "1.9.3" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, + {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"}, + {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b"}, + {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9"}, + {file = "scipy-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523"}, + {file = "scipy-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096"}, + {file = "scipy-1.9.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c"}, + {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab"}, + {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb"}, + {file = "scipy-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31"}, + {file = "scipy-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840"}, + {file = "scipy-1.9.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5"}, + {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108"}, + {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc"}, + {file = "scipy-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e"}, + {file = "scipy-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c"}, + {file = "scipy-1.9.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95"}, + {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e"}, + {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0"}, + {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"}, + {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"}, +] + +[package.dependencies] +numpy = ">=1.18.5,<1.26.0" + +[package.extras] +dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] +doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] +test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + +[[package]] +name = "setuptools" +version = "74.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "setuptools-74.0.0-py3-none-any.whl", hash = "sha256:0274581a0037b638b9fc1c6883cc71c0210865aaa76073f7882376b641b84e8f"}, + {file = "setuptools-74.0.0.tar.gz", hash = "sha256:a85e96b8be2b906f3e3e789adec6a9323abf79758ecfa3065bd740d81158b11e"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.5.2) ; sys_platform != \"cygwin\""] +core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.11.*)", "pytest-mypy"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main"] +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sktime" +version = "0.24.2" +description = "A unified framework for machine learning with time series" +optional = false +python-versions = "<3.13,>=3.8" +groups = ["main"] +files = [ + {file = "sktime-0.24.2-py3-none-any.whl", hash = "sha256:4c47d08fc11eddb77c8f5ca64509802a4972b33e1518aa801c4501472100012d"}, + {file = "sktime-0.24.2.tar.gz", hash = "sha256:587450908035482b8e5bba7344424706e200496c951d8baf4835148333f74168"}, +] + +[package.dependencies] +numpy = ">=1.21,<1.27" +packaging = "*" +pandas = ">=1.1,<2.2.0" +scikit-base = "<0.7.0" +scikit-learn = ">=0.24,<1.4.0" +scipy = ">=1.2,<2.0.0" + +[package.extras] +alignment = ["dtw-python (>=1.3,<1.4)", "numba (>=0.53,<0.59) ; python_version < \"3.12\""] +all-extras = ["arch (>=5.6,<6.3.0)", "cloudpickle", "dash (!=2.9.0)", "dask", "dtw-python", "esig (==0.9.7) ; python_version < \"3.10\"", "filterpy (>=1.4.5) ; python_version < \"3.11\"", "gluonts (>=0.9)", "h5py ; python_version < \"3.12\"", "hmmlearn (>=0.2.7) ; python_version < \"3.11\"", "holidays", "keras-self-attention ; python_version < \"3.11\"", "kotsu (>=0.3.1)", "matplotlib (>=3.3.2)", "mne", "numba (>=0.53,<0.59) ; python_version < \"3.12\"", "pmdarima (>=1.8,!=1.8.1,<3.0.0) ; python_version < \"3.12\"", "prophet (>=1.1) ; python_version < \"3.12\"", "pycatch22 (<0.4.4)", "pykalman-bardo (>=0.9.7,<0.10)", "pyod (>=0.8) ; python_version < \"3.11\"", "scikit-optimize", "scikit-posthocs (>=0.6.5)", "seaborn (>=0.11)", "seasonal", "skpro (>=2,<2.2.0)", "statsforecast (>=0.5.2,<1.7.0) ; python_version < \"3.12\"", "statsmodels (>=0.12.1)", "stumpy (>=1.5.1) ; python_version < \"3.11\"", "tbats (>=1.1) ; python_version < \"3.12\"", "tensorflow ; python_version < \"3.12\"", "tsfresh (>=0.17) ; python_version < \"3.12\"", "tslearn (>=0.5.2,<0.6.0) ; python_version < \"3.11\"", "xarray"] +all-extras-pandas2 = ["arch (>=5.6,<6.3.0)", "cloudpickle", "dash (!=2.9.0)", "dask (<2023.12.2)", "dtw-python", "esig (==0.9.7) ; python_version < \"3.10\"", "filterpy (>=1.4.5) ; python_version < \"3.11\"", "gluonts (>=0.9)", "h5py ; python_version < \"3.12\"", "hmmlearn (>=0.2.7) ; python_version < \"3.11\"", "holidays", "keras-self-attention ; python_version < \"3.11\"", "kotsu (>=0.3.1)", "matplotlib (>=3.3.2)", "mne", "numba (>=0.53,<0.59) ; python_version < \"3.12\"", "pmdarima (>=1.8,!=1.8.1,<3.0.0) ; python_version < \"3.12\"", "prophet (>=1.1) ; python_version < \"3.12\"", "pycatch22 (<0.4.4)", "pykalman-bardo (>=0.9.7,<0.10)", "pyod (>=0.8) ; python_version < \"3.11\"", "scikit-posthocs (>=0.6.5)", "seaborn (>=0.11)", "seasonal", "skpro (>=2,<2.2.0)", "statsforecast (>=0.5.2,<1.7.0) ; python_version < \"3.12\"", "statsmodels (>=0.12.1)", "stumpy (>=1.5.1) ; python_version < \"3.11\"", "tbats (>=1.1) ; python_version < \"3.12\"", "tensorflow ; python_version < \"3.12\"", "tsfresh (>=0.17) ; python_version < \"3.12\"", "tslearn (>=0.5.2,<0.6.0) ; python_version < \"3.11\"", "xarray"] +annotation = ["hmmlearn (>=0.2.7,<0.4)", "numba (>=0.53,<0.59) ; python_version < \"3.12\"", "pyod (>=0.8,<1.2) ; python_version < \"3.12\""] +binder = ["jupyter", "pandas (<2.0.0)"] +classification = ["esig (>=0.9.7,<0.10) ; python_version < \"3.11\"", "numba (>=0.53,<0.59) ; python_version < \"3.12\"", "tensorflow (>=2,<=2.14) ; python_version < \"3.12\"", "tsfresh (>=0.17,<0.21) ; python_version < \"3.12\""] +clustering = ["numba (>=0.53,<0.59) ; python_version < \"3.12\"", "tslearn (>=0.5.2,<0.6.3) ; python_version < \"3.12\""] +cython-extras = ["mrseql", "mrsqm ; python_version < \"3.11\"", "numba (<0.59)"] +dev = ["backoff", "httpx", "pre-commit", "pytest", "pytest-cov", "pytest-randomly", "pytest-timeout", "pytest-xdist", "wheel"] +dl = ["tensorflow (>=2,<=2.14) ; python_version < \"3.12\"", "torch ; python_version < \"3.12\""] +docs = ["Sphinx (!=7.2.0,<8.0.0)", "jupyter", "myst-parser", "nbsphinx (>=0.8.6)", "numpydoc", "pydata-sphinx-theme", "sphinx-copybutton", "sphinx-design (<0.6.0)", "sphinx-gallery (<0.16.0)", "sphinx-issues (<4.0.0)", "tabulate"] +forecasting = ["arch (>=5.6,<6.3)", "pmdarima (>=1.8,!=1.8.1,<2.1) ; python_version < \"3.12\"", "prophet (>=1.1,<1.2)", "skpro (>=2,<2.2)", "statsforecast (>=0.5.2,<1.7) ; python_version < \"3.12\"", "statsmodels (>=0.12.1,<0.15)", "tbats (>=1.1,<1.2) ; python_version < \"3.12\""] +mlflow = ["mlflow"] +mlflow-tests = ["boto3", "botocore", "mlflow", "moto"] +networks = ["keras-self-attention (>=0.51,<0.52)", "tensorflow (>=2,<=2.14) ; python_version < \"3.12\""] +pandas1 = ["pandas (<2.0.0)"] +param-est = ["seasonal (>=0.3.1,<0.4)", "statsmodels (>=0.12.1,<0.15)"] +regression = ["numba (>=0.53,<0.59) ; python_version < \"3.12\"", "tensorflow (>=2,<=2.14) ; python_version < \"3.12\""] +tests = ["pytest (>=7.4,<7.5)", "pytest-cov (>=4.1,<4.2)", "pytest-randomly (>=3.15,<3.16)", "pytest-timeout (>=2.1,<2.3)", "pytest-xdist (>=3.3,<3.6)"] +transformations = ["esig (>=0.9.7,<0.10) ; python_version < \"3.11\"", "filterpy (>=1.4.5,<1.5)", "holidays (>=0.29,<0.40)", "mne (>=1.5,<1.6)", "numba (>=0.53,<0.59) ; python_version < \"3.12\"", "pycatch22 (>=0.4,<0.4.5)", "pykalman-bardo (>=0.9.7,<0.10)", "statsmodels (>=0.12.1,<0.15)", "stumpy (>=1.5.1,<1.13) ; python_version < \"3.12\"", "tsfresh (>=0.17,<0.21) ; python_version < \"3.12\""] + +[[package]] +name = "sqlalchemy" +version = "2.0.32" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0c9045ecc2e4db59bfc97b20516dfdf8e41d910ac6fb667ebd3a79ea54084619"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1467940318e4a860afd546ef61fefb98a14d935cd6817ed07a228c7f7c62f389"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5954463675cb15db8d4b521f3566a017c8789222b8316b1e6934c811018ee08b"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167e7497035c303ae50651b351c28dc22a40bb98fbdb8468cdc971821b1ae533"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27dfb676ac02529fb6e343b3a482303f16e6bc3a4d868b73935b8792edb52d0"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf2360a5e0f7bd75fa80431bf8ebcfb920c9f885e7956c7efde89031695cafb8"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win32.whl", hash = "sha256:306fe44e754a91cd9d600a6b070c1f2fadbb4a1a257b8781ccf33c7067fd3e4d"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:99db65e6f3ab42e06c318f15c98f59a436f1c78179e6a6f40f529c8cc7100b22"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21b053be28a8a414f2ddd401f1be8361e41032d2ef5884b2f31d31cb723e559f"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b178e875a7a25b5938b53b006598ee7645172fccafe1c291a706e93f48499ff5"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723a40ee2cc7ea653645bd4cf024326dea2076673fc9d3d33f20f6c81db83e1d"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295ff8689544f7ee7e819529633d058bd458c1fd7f7e3eebd0f9268ebc56c2a0"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:49496b68cd190a147118af585173ee624114dfb2e0297558c460ad7495f9dfe2"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:acd9b73c5c15f0ec5ce18128b1fe9157ddd0044abc373e6ecd5ba376a7e5d961"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win32.whl", hash = "sha256:9365a3da32dabd3e69e06b972b1ffb0c89668994c7e8e75ce21d3e5e69ddef28"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win_amd64.whl", hash = "sha256:8bd63d051f4f313b102a2af1cbc8b80f061bf78f3d5bd0843ff70b5859e27924"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bab3db192a0c35e3c9d1560eb8332463e29e5507dbd822e29a0a3c48c0a8d92"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:19d98f4f58b13900d8dec4ed09dd09ef292208ee44cc9c2fe01c1f0a2fe440e9"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd33c61513cb1b7371fd40cf221256456d26a56284e7d19d1f0b9f1eb7dd7e8"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6ba0497c1d066dd004e0f02a92426ca2df20fac08728d03f67f6960271feec"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2b6be53e4fde0065524f1a0a7929b10e9280987b320716c1509478b712a7688c"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:916a798f62f410c0b80b63683c8061f5ebe237b0f4ad778739304253353bc1cb"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win32.whl", hash = "sha256:31983018b74908ebc6c996a16ad3690301a23befb643093fcfe85efd292e384d"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win_amd64.whl", hash = "sha256:4363ed245a6231f2e2957cccdda3c776265a75851f4753c60f3004b90e69bfeb"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8afd5b26570bf41c35c0121801479958b4446751a3971fb9a480c1afd85558e"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c750987fc876813f27b60d619b987b057eb4896b81117f73bb8d9918c14f1cad"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0102afff4890f651ed91120c1120065663506b760da4e7823913ebd3258be"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:78c03d0f8a5ab4f3034c0e8482cfcc415a3ec6193491cfa1c643ed707d476f16"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:3bd1cae7519283ff525e64645ebd7a3e0283f3c038f461ecc1c7b040a0c932a1"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win32.whl", hash = "sha256:01438ebcdc566d58c93af0171c74ec28efe6a29184b773e378a385e6215389da"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:4979dc80fbbc9d2ef569e71e0896990bc94df2b9fdbd878290bd129b65ab579c"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c742be912f57586ac43af38b3848f7688863a403dfb220193a882ea60e1ec3a"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:62e23d0ac103bcf1c5555b6c88c114089587bc64d048fef5bbdb58dfd26f96da"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:251f0d1108aab8ea7b9aadbd07fb47fb8e3a5838dde34aa95a3349876b5a1f1d"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef18a84e5116340e38eca3e7f9eeaaef62738891422e7c2a0b80feab165905f"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3eb6a97a1d39976f360b10ff208c73afb6a4de86dd2a6212ddf65c4a6a2347d5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0c1c9b673d21477cec17ab10bc4decb1322843ba35b481585facd88203754fc5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win32.whl", hash = "sha256:c41a2b9ca80ee555decc605bd3c4520cc6fef9abde8fd66b1cf65126a6922d65"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:8a37e4d265033c897892279e8adf505c8b6b4075f2b40d77afb31f7185cd6ecd"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fec964fba2ef46476312a03ec8c425956b05c20220a1a03703537824b5e8e1"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:328429aecaba2aee3d71e11f2477c14eec5990fb6d0e884107935f7fb6001632"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85a01b5599e790e76ac3fe3aa2f26e1feba56270023d6afd5550ed63c68552b3"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf04784797dcdf4c0aa952c8d234fa01974c4729db55c45732520ce12dd95b4"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4488120becf9b71b3ac718f4138269a6be99a42fe023ec457896ba4f80749525"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:14e09e083a5796d513918a66f3d6aedbc131e39e80875afe81d98a03312889e6"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win32.whl", hash = "sha256:0d322cc9c9b2154ba7e82f7bf25ecc7c36fbe2d82e2933b3642fc095a52cfc78"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:7dd8583df2f98dea28b5cd53a1beac963f4f9d087888d75f22fcc93a07cf8d84"}, + {file = "SQLAlchemy-2.0.32-py3-none-any.whl", hash = "sha256:e567a8793a692451f706b363ccf3c45e056b67d90ead58c3bc9471af5d212202"}, + {file = "SQLAlchemy-2.0.32.tar.gz", hash = "sha256:c1b88cc8b02b6a5f0efb0345a03672d4c897dc7d92585176f88c67346f565ea8"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "statsmodels" +version = "0.14.1" +description = "Statistical computations and models for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "statsmodels-0.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:43af9c0b07c9d72f275cf14ea54a481a3f20911f0b443181be4769def258fdeb"}, + {file = "statsmodels-0.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a16975ab6ad505d837ba9aee11f92a8c5b49c4fa1ff45b60fe23780b19e5705e"}, + {file = "statsmodels-0.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e278fe74da5ed5e06c11a30851eda1af08ef5af6be8507c2c45d2e08f7550dde"}, + {file = "statsmodels-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0564d92cb05b219b4538ed09e77d96658a924a691255e1f7dd23ee338df441b"}, + {file = "statsmodels-0.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5385e22e72159a09c099c4fb975f350a9f3afeb57c1efce273b89dcf1fe44c0f"}, + {file = "statsmodels-0.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:0a8aae75a2e08ebd990e5fa394f8e32738b55785cb70798449a3f4207085e667"}, + {file = "statsmodels-0.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b69a63ad6c979a6e4cde11870ffa727c76a318c225a7e509f031fbbdfb4e416a"}, + {file = "statsmodels-0.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7562cb18a90a114f39fab6f1c25b9c7b39d9cd5f433d0044b430ca9d44a8b52c"}, + {file = "statsmodels-0.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3abaca4b963259a2bf349c7609cfbb0ce64ad5fb3d92d6f08e21453e4890248"}, + {file = "statsmodels-0.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0f727fe697f6406d5f677b67211abe5a55101896abdfacdb3f38410405f6ad8"}, + {file = "statsmodels-0.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b6838ac6bdb286daabb5e91af90fd4258f09d0cec9aace78cc441cb2b17df428"}, + {file = "statsmodels-0.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:709bfcef2dbe66f705b17e56d1021abad02243ee1a5d1efdb90f9bad8b06a329"}, + {file = "statsmodels-0.14.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f32a7cd424cf33304a54daee39d32cccf1d0265e652c920adeaeedff6d576457"}, + {file = "statsmodels-0.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f8c30181c084173d662aaf0531867667be2ff1bee103b84feb64f149f792dbd2"}, + {file = "statsmodels-0.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de2b97413913d52ad6342dece2d653e77f78620013b7705fad291d4e4266ccb"}, + {file = "statsmodels-0.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3420f88289c593ba2bca33619023059c476674c160733bd7d858564787c83d3"}, + {file = "statsmodels-0.14.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c008e16096f24f0514e53907890ccac6589a16ad6c81c218f2ee6752fdada555"}, + {file = "statsmodels-0.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:bc0351d279c4e080f0ce638a3d886d312aa29eade96042e3ba0a73771b1abdfb"}, + {file = "statsmodels-0.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf293ada63b2859d95210165ad1dfcd97bd7b994a5266d6fbeb23659d8f0bf68"}, + {file = "statsmodels-0.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44ca8cb88fa3d3a4ffaff1fb8eb0e98bbf83fc936fcd9b9eedee258ecc76696a"}, + {file = "statsmodels-0.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d5373d176239993c095b00d06036690a50309a4e00c2da553b65b840f956ae6"}, + {file = "statsmodels-0.14.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a532dfe899f8b6632cd8caa0b089b403415618f51e840d1817a1e4b97e200c73"}, + {file = "statsmodels-0.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:4fe0a60695952b82139ae8750952786a700292f9e0551d572d7685070944487b"}, + {file = "statsmodels-0.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:04293890f153ffe577e60a227bd43babd5f6c1fc50ea56a3ab1862ae85247a95"}, + {file = "statsmodels-0.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e70a2e93d54d40b2cb6426072acbc04f35501b1ea2569f6786964adde6ca572"}, + {file = "statsmodels-0.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab3a73d16c0569adbba181ebb967e5baaa74935f6d2efe86ac6fc5857449b07d"}, + {file = "statsmodels-0.14.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eefa5bcff335440ee93e28745eab63559a20cd34eea0375c66d96b016de909b3"}, + {file = "statsmodels-0.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:bc43765710099ca6a942b5ffa1bac7668965052542ba793dd072d26c83453572"}, + {file = "statsmodels-0.14.1.tar.gz", hash = "sha256:2260efdc1ef89f39c670a0bd8151b1d0843567781bcafec6cda0534eb47a94f6"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.18,<2"}, + {version = ">=1.22.3,<2", markers = "python_version == \"3.10\" and platform_system == \"Windows\" and platform_python_implementation != \"PyPy\""}, +] +packaging = ">=21.3" +pandas = ">=1.0,<2.1.0 || >2.1.0" +patsy = ">=0.5.4" +scipy = ">=1.4,<1.9.2 || >1.9.2" + +[package.extras] +build = ["cython (>=0.29.33)"] +develop = ["colorama", "cython (>=0.29.33)", "cython (>=0.29.33,<4.0.0)", "flake8", "isort", "joblib", "matplotlib (>=3)", "oldest-supported-numpy (>=2022.4.18)", "pytest (>=7.3.0)", "pytest-cov", "pytest-randomly", "pytest-xdist", "pywinpty ; os_name == \"nt\"", "setuptools-scm[toml] (>=8.0,<9.0)"] +docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"] + +[[package]] +name = "sympy" +version = "1.13.2" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "sympy-1.13.2-py3-none-any.whl", hash = "sha256:c51d75517712f1aed280d4ce58506a4a88d635d6b5dd48b39102a7ae1f3fcfe9"}, + {file = "sympy-1.13.2.tar.gz", hash = "sha256:401449d84d07be9d0c7a46a64bd54fe097667d5e7181bfe67ec777be9e01cb13"}, +] + +[package.dependencies] +mpmath = ">=1.1.0,<1.4" + +[package.extras] +dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] + +[[package]] +name = "threadpoolctl" +version = "3.5.0" +description = "threadpoolctl" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, + {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, +] + +[[package]] +name = "thrift" +version = "0.13.0" +description = "Python bindings for the Apache Thrift RPC system" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "thrift-0.13.0.tar.gz", hash = "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89"}, +] + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "torch" +version = "2.2.0" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +files = [ + {file = "torch-2.2.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:d366158d6503a3447e67f8c0ad1328d54e6c181d88572d688a625fac61b13a97"}, + {file = "torch-2.2.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:707f2f80402981e9f90d0038d7d481678586251e6642a7a6ef67fc93511cb446"}, + {file = "torch-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:15c8f0a105c66b28496092fca1520346082e734095f8eaf47b5786bac24b8a31"}, + {file = "torch-2.2.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:0ca4df4b728515ad009b79f5107b00bcb2c63dc202d991412b9eb3b6a4f24349"}, + {file = "torch-2.2.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:3d3eea2d5969b9a1c9401429ca79efc668120314d443d3463edc3289d7f003c7"}, + {file = "torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:0d1c580e379c0d48f0f0a08ea28d8e373295aa254de4f9ad0631f9ed8bc04c24"}, + {file = "torch-2.2.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9328e3c1ce628a281d2707526b4d1080eae7c4afab4f81cea75bde1f9441dc78"}, + {file = "torch-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:03c8e660907ac1b8ee07f6d929c4e15cd95be2fb764368799cca02c725a212b8"}, + {file = "torch-2.2.0-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:da0cefe7f84ece3e3b56c11c773b59d1cb2c0fd83ddf6b5f7f1fd1a987b15c3e"}, + {file = "torch-2.2.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:f81d23227034221a4a4ff8ef24cc6cec7901edd98d9e64e32822778ff01be85e"}, + {file = "torch-2.2.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:dcbfb2192ac41ca93c756ebe9e2af29df0a4c14ee0e7a0dd78f82c67a63d91d4"}, + {file = "torch-2.2.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:9eeb42971619e24392c9088b5b6d387d896e267889d41d267b1fec334f5227c5"}, + {file = "torch-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:c718b2ca69a6cac28baa36d86d8c0ec708b102cebd1ceb1b6488e404cd9be1d1"}, + {file = "torch-2.2.0-cp312-none-macosx_10_9_x86_64.whl", hash = "sha256:f11d18fceb4f9ecb1ac680dde7c463c120ed29056225d75469c19637e9f98d12"}, + {file = "torch-2.2.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:ee1da852bfd4a7e674135a446d6074c2da7194c1b08549e31eae0b3138c6b4d2"}, + {file = "torch-2.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0d819399819d0862268ac531cf12a501c253007df4f9e6709ede8a0148f1a7b8"}, + {file = "torch-2.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:08f53ccc38c49d839bc703ea1b20769cc8a429e0c4b20b56921a9f64949bf325"}, + {file = "torch-2.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:93bffe3779965a71dab25fc29787538c37c5d54298fd2f2369e372b6fb137d41"}, + {file = "torch-2.2.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:c17ec323da778efe8dad49d8fb534381479ca37af1bfc58efdbb8607a9d263a3"}, + {file = "torch-2.2.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:c02685118008834e878f676f81eab3a952b7936fa31f474ef8a5ff4b5c78b36d"}, + {file = "torch-2.2.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d9f39d6f53cec240a0e3baa82cb697593340f9d4554cee6d3d6ca07925c2fac0"}, + {file = "torch-2.2.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:51770c065206250dc1222ea7c0eff3f88ab317d3e931cca2aee461b85fbc2472"}, + {file = "torch-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:008e4c6ad703de55af760c73bf937ecdd61a109f9b08f2bbb9c17e7c7017f194"}, + {file = "torch-2.2.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:de8680472dd14e316f42ceef2a18a301461a9058cd6e99a1f1b20f78f11412f1"}, + {file = "torch-2.2.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:99e1dcecb488e3fd25bcaac56e48cdb3539842904bdc8588b0b255fde03a254c"}, +] + +[package.dependencies] +filelock = "*" +fsspec = "*" +jinja2 = "*" +networkx = "*" +nvidia-cublas-cu12 = {version = "12.1.3.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "8.9.2.26", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.0.2.54", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.2.106", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.4.5.107", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.1.0.106", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +sympy = "*" +triton = {version = "2.2.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +typing-extensions = ">=4.8.0" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] +optree = ["optree (>=0.9.1)"] + +[[package]] +name = "tqdm" +version = "4.66.5" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, + {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "triton" +version = "2.2.0" +description = "A language and compiler for custom Deep Learning operations" +optional = false +python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "triton-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2294514340cfe4e8f4f9e5c66c702744c4a117d25e618bd08469d0bfed1e2e5"}, + {file = "triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da58a152bddb62cafa9a857dd2bc1f886dbf9f9c90a2b5da82157cd2b34392b0"}, + {file = "triton-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af58716e721460a61886668b205963dc4d1e4ac20508cc3f623aef0d70283d5"}, + {file = "triton-2.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8fe46d3ab94a8103e291bd44c741cc294b91d1d81c1a2888254cbf7ff846dab"}, + {file = "triton-2.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8ce26093e539d727e7cf6f6f0d932b1ab0574dc02567e684377630d86723ace"}, + {file = "triton-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:227cc6f357c5efcb357f3867ac2a8e7ecea2298cd4606a8ba1e931d1d5a947df"}, +] + +[package.dependencies] +filelock = "*" + +[package.extras] +build = ["cmake (>=3.20)", "lit"] +tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)", "torch"] +tutorials = ["matplotlib", "pandas", "tabulate", "torch"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "zipp" +version = "3.20.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.9\"" +files = [ + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.8, <3.13" +content-hash = "f8ff25befae83d79c99b9eb13009b72fcbb717da26800d179eaf807f19a747f7" diff --git a/iotdb-core/ainode/pom.xml b/iotdb-core/ainode/pom.xml new file mode 100644 index 0000000000000..e38b9fcda0821 --- /dev/null +++ b/iotdb-core/ainode/pom.xml @@ -0,0 +1,371 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-core + 2.0.4-SNAPSHOT + + iotdb-ainode + IoTDB: Core: AINode + + + + org.apache.iotdb + iotdb-thrift-commons + 2.0.4-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-thrift + 2.0.4-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-thrift-confignode + 2.0.4-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-thrift-ainode + 2.0.4-SNAPSHOT + provided + + + + + + + org.apache.maven.plugins + maven-clean-plugin + + + + dist + + + ainode + + conf/ + thrift/ + + + + target + + + venv + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + org.codehaus.gmaven + groovy-maven-plugin + 2.1.1 + + + + sync-python-version + validate + + execute + + + ${project.basedir}/resources/syncPythonVersion.groovy + + + + + + org.apache.groovy + groovy + 4.0.22 + + + org.apache.groovy + groovy-toml + 4.0.22 + + + + + + org.apache.maven.plugins + maven-resources-plugin + + ${project.build.sourceEncoding} + + + + copy-thrift-python-resources + generate-sources + + copy-resources + + + ${basedir}/ainode/thrift/ + + + ${basedir}/../../iotdb-protocol/thrift-commons/target/generated-sources-python/iotdb/thrift/ + + + ${basedir}/../../iotdb-protocol/thrift-confignode/target/generated-sources-python/iotdb/thrift/ + + + ${basedir}/../../iotdb-protocol/thrift-ainode/target/generated-sources-python/iotdb/thrift/ + + + ${basedir}/../../iotdb-protocol/thrift-datanode/target/generated-sources-python/iotdb/thrift/ + rpc/** + + + + + + + copy-pom-properties + generate-sources + + copy-resources + + + ${basedir}/ainode/conf/ + + + ${basedir}/resources/ + pom.properties + true + + + + + + + + + pl.project13.maven + git-commit-id-plugin + + + generate-git-properties + generate-resources + + revision + + + true + ${project.basedir}/ainode/conf/git.properties + + ^git.commit.id.abbrev$ + ^git.dirty$ + + full + false + true + + -dev + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + python-venv + initialize + + exec + + + ${python.exe.bin} + + -m + venv + ./venv + + + + + + python-upgrade-pip + initialize + + exec + + + ${python.venv.bin}${python.exe.bin} + + -m + pip + install + --upgrade + pip + + + + + + python-install-poetry + initialize + + exec + + + ${python.venv.bin}pip3 + + install + poetry + + + + + + + python-compile + compile + + exec + + + ${python.venv.bin}poetry + + build + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + create-ainode-zip + package + + single + + + apache-iotdb-ainode-${project.version} + false + + ainode.xml + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + check-dependencies + + analyze-only + + verify + + + org.apache.iotdb:iotdb-thrift-commons + org.apache.iotdb:iotdb-thrift + org.apache.iotdb:iotdb-thrift-confignode + org.apache.iotdb:iotdb-thrift-ainode + + + + + + + + diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml new file mode 100644 index 0000000000000..70557d4f2c041 --- /dev/null +++ b/iotdb-core/ainode/pyproject.toml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "apache-iotdb-ainode" +version = "2.0.0.dev" +description = "Apache IoTDB AINode" +readme = "README.md" +authors = ["Apache Software Foundation "] +license = "Apache License, Version 2.0" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] +include = [ + {path = "ainode/thrift/*", format = "wheel"}, + {path = "ainode/thrift/common/*", format = "wheel"}, + {path = "ainode/thrift/confignode/*", format = "wheel"}, + {path = "ainode/thrift/datanode/*", format = "wheel"}, + {path = "ainode/thrift/ainode/*", format = "wheel"}, + {path = "ainode/conf/*", format = "wheel"}, +] +packages = [ + { include = "ainode" } +] + +[tool.poetry.dependencies] +python = ">=3.8, <3.13" +numpy = "^1.21.4" +pandas = "^1.3.5" +torch = ">=2.2.0" +pylru = "^1.2.1" +thrift = ">=0.14.0" +dynaconf = "^3.1.11" +requests = "^2.31.0" +optuna = "^3.2.0" +psutil = "^5.9.5" +sktime = "^0.24.1" +pmdarima = "^2.0.4" +hmmlearn = "^0.3.0" +apache-iotdb = "2.0.4.dev0" +einops = "^0.8.1" +safetensors = "^0.5.1" +huggingface_hub = "^0.30.1" + +[tool.poetry.scripts] +ainode = "ainode.core.script:main" diff --git a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties new file mode 100644 index 0000000000000..2b208e7212538 --- /dev/null +++ b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Used for indicate cluster name and distinguish different cluster. +# Datatype: string +# cluster_name=defaultCluster + +# ConfigNode address registered at AINode startup +# Allow modifications only before starting the service for the first time +# Datatype: String +# ain_seed_config_node=127.0.0.1:10710 + +# Used for connection of DataNode/ConfigNode clients +# Could set 127.0.0.1(for local test) or ipv4 address +# Datatype: String +# ain_inference_rpc_address=127.0.0.1 + +# Used for connection of DataNode/ConfigNode clients +# Bind with MN_RPC_ADDRESS +# Datatype: String +# ain_inference_rpc_port=10810 + +# The AINode metadata storage path. +# The starting directory of the relative path is related to the operating system. +# It is recommended to use an absolute path. +# Datatype: String +# ain_system_dir=data/ainode/system + +# The path where AINode stores model files +# The starting directory of the relative path is related to the operating system. +# It is recommended to use an absolute path. +# Datatype: String +# ain_models_dir=data/ainode/models + +# The path where AINode stores logs +# The starting directory of the relative path is related to the operating system. +# It is recommended to use an absolute path. +# Datatype: String +# ain_logs_dir=logs/ainode + +# Whether to use compression in Thrift +# Please use 0 or 1 +# Datatype: Boolean +# ain_thrift_compression_enabled=0 \ No newline at end of file diff --git a/iotdb-core/ainode/resources/pom.properties b/iotdb-core/ainode/resources/pom.properties new file mode 100644 index 0000000000000..ee2e2a650e513 --- /dev/null +++ b/iotdb-core/ainode/resources/pom.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +artifactId=iotdb-ainode +groupId=org.apache.iotdb +version=${project.version} \ No newline at end of file diff --git a/iotdb-core/ainode/resources/syncPythonVersion.groovy b/iotdb-core/ainode/resources/syncPythonVersion.groovy new file mode 100644 index 0000000000000..373bfd7fa0a32 --- /dev/null +++ b/iotdb-core/ainode/resources/syncPythonVersion.groovy @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import groovy.toml.TomlSlurper + +import java.util.regex.Matcher + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// The entire Python "check" block is borrowed from Apache PLC4X's build. +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +allConditionsMet = true + +/** + * Version extraction function/macro. It looks for occurrence of x.y or x.y.z + * in passed input text (likely output from `program --version` command if found). + * + * @param input + * @return + */ +private static Matcher extractVersion(input) { + def matcher = input =~ /(\d+\.\d+(\.\d+)?).*/ + matcher +} + +def checkVersionAtLeast(String current, String minimum) { + def currentSegments = current.tokenize('.') + def minimumSegments = minimum.tokenize('.') + def numSegments = Math.min(currentSegments.size(), minimumSegments.size()) + for (int i = 0; i < numSegments; ++i) { + def currentSegment = currentSegments[i].toInteger() + def minimumSegment = minimumSegments[i].toInteger() + if (currentSegment < minimumSegment) { + println current.padRight(14) + " FAILED (required min " + minimum + " but got " + current + ")" + return false + } else if (currentSegment > minimumSegment) { + println current.padRight(14) + " OK" + return true + } + } + def curNotShorter = currentSegments.size() >= minimumSegments.size() + if (curNotShorter) { + println current.padRight(14) + " OK" + } else { + println current.padRight(14) + " (required min " + minimum + " but got " + current + ")" + } + curNotShorter +} + +def checkVersionAtMost(String current, String maximum) { + def currentSegments = current.tokenize('.') + def maximumSegments = maximum.tokenize('.') + def numSegments = Math.min(currentSegments.size(), maximumSegments.size()) + for (int i = 0; i < numSegments; ++i) { + def currentSegment = currentSegments[i].toInteger() + def maximumSegment = maximumSegments[i].toInteger() + if (currentSegment > maximumSegment) { + println current.padRight(14) + " FAILED (required max " + maximum + " but got " + current + ")" + return false + } else if (currentSegment < maximumSegment) { + println current.padRight(14) + " OK" + return true + } + } + def curNotShorter = currentSegments.size() >= maximumSegments.size() + if (curNotShorter) { + println current.padRight(14) + " OK" + } else { + println current.padRight(14) + " (required max " + maximum + " but got " + current + ")" + } + curNotShorter +} + +def checkPython() { + String python = project.properties['python.exe.bin'] + println "Using python executable: " + python.padRight(14) + " OK" + print "Detecting Python version: " + try { + def process = (python + " --version").execute() + def stdOut = new StringBuilder() + def stdErr = new StringBuilder() + process.waitForProcessOutput(stdOut, stdErr) + Matcher matcher = extractVersion(stdOut + stdErr) + if (matcher.size() > 0) { + String curVersion = matcher[0][1] + def result = checkVersionAtLeast(curVersion, "3.8.0") + if (!result) { + allConditionsMet = false + } + result = checkVersionAtMost(curVersion, "3.13") + if (!result) { + allConditionsMet = false + } + } else { + println "missing (Please install at least version 3.8.0 and at most one of the 3.13.x versions)" + allConditionsMet = false + } + } catch (Exception ignored) { + println "missing" + println "--- output of version `${python} --version` command ---" + println output + println "----------------------------------------------------" + allConditionsMet = false + } +} + + +// On Ubuntu it seems that venv is generally available, but the 'ensurepip' command fails. +// In this case we need to install the python3-venv package. Unfortunately checking the +// venv is successful in this case, so we need this slightly odd test. +def checkPythonVenv() { + print "Detecting venv: " + try { + def python = project.properties['python.exe.bin'] + def cmdArray = [python, "-Im", "ensurepip"] + def process = cmdArray.execute() + def stdOut = new StringBuilder() + def stdErr = new StringBuilder() + process.waitForProcessOutput(stdOut, stdErr) + if (stdErr.contains("No module named")) { + println "missing" + println "--- output of version `python -Im \"ensurepip\"` command ---" + println output + println "------------------------------------------------------------" + allConditionsMet = false + } else { + println " OK" + } + } catch (Exception e) { + println "missing" + println "--- failed with exception ---" + println e + e.printStackTrace() + println "----------------------------------------------------" + allConditionsMet = false + } +} + +// Check the python environment is setup correctly. +checkPython() +checkPythonVenv() + +if (!allConditionsMet) { + throw new RuntimeException("Not all conditions met, see log for details.") +} +println "" +println "All known conditions met successfully." +println "" + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Calculate the version that we should use in the python build. +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +def currentMavenVersion = project.version as String +def currentPythonVersion = currentMavenVersion +if(currentMavenVersion.contains("-SNAPSHOT")) { + currentPythonVersion = currentMavenVersion.split("-SNAPSHOT")[0] + ".dev" +} +println "Current Project Version in Maven: " + currentMavenVersion +println "Current Project Version in Python: " + currentPythonVersion + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Synchronize the version in pyproject.toml and the one used in the maven pom. +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +def pyprojectFile = new File(project.basedir, "pyproject.toml") +def ts = new TomlSlurper() +def toml = ts.parse(pyprojectFile) +def pyprojectFileVersion = toml.tool.poetry.version +if (pyprojectFileVersion != currentPythonVersion) { + pyprojectFile.text = pyprojectFile.text.replace("version = \"" + pyprojectFileVersion + "\"", "version = \"" + currentPythonVersion + "\"") + println "Version in pyproject.toml updated from " + pyprojectFileVersion + " to " + currentPythonVersion + // TODO: When releasing, we might need to manually add this file to the release preparation commit. +} else { + println "Version in pyproject.toml is up to date" +} \ No newline at end of file diff --git a/iotdb-core/antlr/pom.xml b/iotdb-core/antlr/pom.xml index 5247d212fe3c7..86f2add9a6357 100644 --- a/iotdb-core/antlr/pom.xml +++ b/iotdb-core/antlr/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-antlr IoTDB: Core: Antlr-Parser diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index c9cfc400dcee7..2d8f2fa1ce742 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -53,14 +53,17 @@ keyWords | BOUNDARY | BY | CACHE + | CALL | CASE | CAST | CHILD | CLEAR | CLUSTER | CLUSTERID + | COMMENT | CONCAT | CONDITION + | CONFIGNODE | CONFIGNODES | CONFIGURATION | CONNECTION @@ -72,16 +75,20 @@ keyWords | CQ | CQS | CREATE + | CURRENT_SQL_DIALECT + | CURRENT_USER | DATA | DATA_REPLICATION_FACTOR | DATA_REGION_GROUP_NUM | DATABASE | DATABASES + | DATANODE | DATANODEID | DATANODES | DATASET | DEACTIVATE | DEBUG + | DEFAULT | DELETE | DESC | DESCRIBE @@ -95,10 +102,12 @@ keyWords | ELSE | END | ENDTIME + | ESCAPE | EVERY | EXPLAIN | EXTRACTOR | FALSE + | FIELD | FILL | FILE | FIRST @@ -112,9 +121,11 @@ keyWords | GRANT | GROUP | HAVING + | HEAD | HYPERPARAMETERS | IN | INDEX + | INFERENCE | INFO | INSERT | INTO @@ -135,6 +146,9 @@ keyWords | MERGE | METADATA | MIGRATE + | AINODES + | MODEL + | MODELS | MODIFY | NAN | NODEID @@ -187,6 +201,7 @@ keyWords | RESOURCE | REPAIR | REPLACE + | RESTRICT | REVOKE | ROLE | ROUND @@ -206,6 +221,7 @@ keyWords | SOFFSET | SOURCE | SPACE + | SQL_DIALECT | STORAGE | START | STARTTIME @@ -213,10 +229,14 @@ keyWords | STATELESS | STATEMENT | STOP + | SUBSCRIPTION | SUBSCRIPTIONS | SUBSTRING | SYSTEM + | TABLE + | TAG | TAGS + | TAIL | TASK | TEMPLATE | TEMPLATES @@ -233,6 +253,7 @@ keyWords | TOPIC | TOPICS | TRACING + | TREE | TRIGGER | TRIGGERS | TRUE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 8986fce18e145..6a0c97ec32160 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -56,19 +56,23 @@ ddlStatement | createPipe | alterPipe | dropPipe | startPipe | stopPipe | showPipes // Pipe Plugin | createPipePlugin | dropPipePlugin | showPipePlugins - // TOPIC - | createTopic | dropTopic | showTopics // Subscription - | showSubscriptions + | createTopic | dropTopic | showTopics | showSubscriptions | dropSubscription // CQ | createContinuousQuery | dropContinuousQuery | showContinuousQueries // Cluster | showVariables | showCluster | showRegions | showDataNodes | showConfigNodes | showClusterId - | getRegionId | getTimeSlotList | countTimeSlotList | getSeriesSlotList | migrateRegion | verifyConnection + | getRegionId | getTimeSlotList | countTimeSlotList | getSeriesSlotList + | migrateRegion | reconstructRegion | extendRegion | removeRegion | removeDataNode | removeConfigNode + | verifyConnection + // AINode + | showAINodes | createModel | dropModel | showModels | callInference // Quota | setSpaceQuota | showSpaceQuota | setThrottleQuota | showThrottleQuota // View | createLogicalView | dropLogicalView | showLogicalView | renameLogicalView | alterLogicalView + // Table View + | createTableView ; dmlStatement @@ -86,7 +90,7 @@ utilityStatement | setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile - | removeFile | unloadFile + | removeFile | unloadFile | setSqlDialect | showCurrentSqlDialect | showCurrentUser ; /** @@ -110,8 +114,6 @@ databaseAttributeClause databaseAttributeKey : TTL - | SCHEMA_REPLICATION_FACTOR - | DATA_REPLICATION_FACTOR | TIME_PARTITION_INTERVAL | SCHEMA_REGION_GROUP_NUM | DATA_REGION_GROUP_NUM @@ -489,6 +491,11 @@ showConfigNodes : SHOW CONFIGNODES ; +// ---- Show AI Nodes +showAINodes + : SHOW AINODES + ; + // ---- Show Cluster Id showClusterId : SHOW CLUSTERID @@ -529,16 +536,39 @@ migrateRegion : MIGRATE REGION regionId=INTEGER_LITERAL FROM fromId=INTEGER_LITERAL TO toId=INTEGER_LITERAL ; +reconstructRegion + : RECONSTRUCT REGION regionIds+=INTEGER_LITERAL (COMMA regionIds+=INTEGER_LITERAL)* ON targetDataNodeId=INTEGER_LITERAL + ; + +extendRegion + : EXTEND REGION regionId=INTEGER_LITERAL TO targetDataNodeId=INTEGER_LITERAL + ; + +removeRegion + : REMOVE REGION regionId=INTEGER_LITERAL FROM targetDataNodeId=INTEGER_LITERAL + ; + verifyConnection : VERIFY CONNECTION (DETAILS)? ; +// ---- Remove DataNode +removeDataNode + : REMOVE DATANODE dataNodeId=INTEGER_LITERAL + ; + +// ---- Remove ConfigNode +removeConfigNode + : REMOVE CONFIGNODE configNodeId=INTEGER_LITERAL + ; + // Pipe Task ========================================================================================= createPipe : CREATE PIPE (IF NOT EXISTS)? pipeName=identifier - extractorAttributesClause? + ((extractorAttributesClause? processorAttributesClause? - connectorAttributesClause + connectorAttributesClause) + |connectorAttributesWithoutWithSinkClause) ; extractorAttributesClause @@ -570,6 +600,10 @@ connectorAttributesClause RR_BRACKET ; +connectorAttributesWithoutWithSinkClause + : LR_BRACKET (connectorAttributeClause COMMA)* connectorAttributeClause? RR_BRACKET + ; + connectorAttributeClause : connectorKey=STRING_LITERAL OPERATOR_SEQ connectorValue=STRING_LITERAL ; @@ -631,7 +665,8 @@ showPipePlugins : SHOW PIPEPLUGINS ; -// Topic ========================================================================================= + +// Subscription ========================================================================================= createTopic : CREATE TOPIC (IF NOT EXISTS)? topicName=identifier topicAttributesClause? ; @@ -652,11 +687,63 @@ showTopics : SHOW ((TOPIC topicName=identifier) | TOPICS ) ; -// Subscriptions ========================================================================================= showSubscriptions : SHOW SUBSCRIPTIONS (ON topicName=identifier)? ; +dropSubscription + : DROP SUBSCRIPTION (IF EXISTS)? subscriptionId=identifier + ; + +// AI Model ========================================================================================= +// ---- Create Model +createModel + : CREATE MODEL modelName=identifier uriClause + | CREATE MODEL modelType=identifier modelId=identifier (WITH HYPERPARAMETERS LR_BRACKET hparamPair (COMMA hparamPair)* RR_BRACKET)? (FROM MODEL existingModelId=identifier)? ON DATASET LR_BRACKET trainingData RR_BRACKET + ; + +trainingData + : dataElement(COMMA dataElement)* + ; + +dataElement + : pathPatternElement (LR_BRACKET timeRange RR_BRACKET)? + ; + +pathPatternElement + : PATH path=prefixPath + ; + +windowFunction + : TAIL LR_BRACKET windowSize=INTEGER_LITERAL RR_BRACKET + | HEAD LR_BRACKET windowSize=INTEGER_LITERAL RR_BRACKET + | COUNT LR_BRACKET interval=INTEGER_LITERAL COMMA step=INTEGER_LITERAL RR_BRACKET + ; + +callInference + : CALL INFERENCE LR_BRACKET modelId=identifier COMMA inputSql=STRING_LITERAL (COMMA hparamPair)* RR_BRACKET + ; + +hparamPair + : hparamKey=attributeKey operator_eq hparamValue + ; + +hparamValue + : attributeValue + | windowFunction + ; + +// ---- Drop Model +dropModel + : DROP MODEL modelId=identifier + ; + +// ---- Show Models +showModels + : SHOW MODELS + | SHOW MODELS modelId=identifier + ; + // Create Logical View createLogicalView : CREATE VIEW viewTargetPaths AS viewSourcePaths @@ -694,6 +781,61 @@ viewSourcePaths | selectClause fromClause ; +// Table view +createTableView + : CREATE (OR REPLACE)? VIEW qualifiedName + LR_BRACKET (viewColumnDefinition (COMMA viewColumnDefinition)*)? RR_BRACKET + comment? + (RESTRICT)? + (WITH properties)? + AS prefixPath + ; + +viewColumnDefinition + : identifier columnCategory=(TAG | TIME | FIELD) comment? + | identifier type (columnCategory=(TAG | TIME | FIELD))? comment? + | identifier (type)? (columnCategory=FIELD)? FROM original_measurement=identifier comment? + ; + +type + : identifier (LR_BRACKET typeParameter (COMMA typeParameter)* RR_BRACKET)? #genericType + ; + +typeParameter + : INTEGER_LITERAL | type + ; + +qualifiedName + : identifier (DOT identifier)* + ; + +properties + : LR_BRACKET propertyAssignments RR_BRACKET + ; + +propertyAssignments + : property (COMMA property)* + ; + +property + : identifier OPERATOR_SEQ propertyValue + ; + +comment + : COMMENT STRING_LITERAL + ; + +propertyValue + : DEFAULT #defaultPropertyValue + | literalExpression #nonDefaultPropertyValue + ; + +// Currently only support this in table property values +literalExpression + : INTEGER_LITERAL + | STRING_LITERAL + ; + /** * 3. Data Manipulation Language (DML) */ @@ -987,7 +1129,7 @@ flush // Clear Cache clearCache - : CLEAR CACHE (ON (LOCAL | CLUSTER))? + : CLEAR (SCHEMA | QUERY | ALL)? CACHE (ON (LOCAL | CLUSTER))? ; // Set Configuration @@ -1085,7 +1227,7 @@ loadTimeseries // Load TsFile loadFile - : LOAD fileName=STRING_LITERAL loadFileAttributeClauses? + : LOAD fileName=STRING_LITERAL ((loadFileAttributeClauses?) | (loadFileWithAttributeClauses)) ; loadFileAttributeClauses @@ -1098,6 +1240,17 @@ loadFileAttributeClause | ONSUCCESS operator_eq (DELETE|NONE) ; +loadFileWithAttributeClauses + : WITH + LR_BRACKET + (loadFileWithAttributeClause COMMA)* loadFileWithAttributeClause? + RR_BRACKET + ; + +loadFileWithAttributeClause + : loadFileWithKey=STRING_LITERAL OPERATOR_SEQ loadFileWithValue=STRING_LITERAL + ; + // Remove TsFile removeFile : REMOVE fileName=STRING_LITERAL @@ -1108,6 +1261,18 @@ unloadFile : UNLOAD srcFileName=STRING_LITERAL dstFileDir=STRING_LITERAL ; +setSqlDialect + : SET SQL_DIALECT OPERATOR_SEQ (TABLE | TREE) + ; + +showCurrentSqlDialect + : SHOW CURRENT_SQL_DIALECT + ; + +showCurrentUser + : SHOW CURRENT_USER + ; + // attribute clauses syncAttributeClauses : attributePair (COMMA? attributePair)* @@ -1215,7 +1380,8 @@ expression | leftExpression=expression (STAR | DIV | MOD) rightExpression=expression | leftExpression=expression (PLUS | MINUS) rightExpression=expression | leftExpression=expression (OPERATOR_GT | OPERATOR_GTE | OPERATOR_LT | OPERATOR_LTE | OPERATOR_SEQ | OPERATOR_DEQ | OPERATOR_NEQ) rightExpression=expression - | unaryBeforeRegularOrLikeExpression=expression operator_not? (REGEXP | LIKE) STRING_LITERAL + | unaryBeforeRegularOrLikeExpression=expression operator_not? REGEXP pattern=STRING_LITERAL + | unaryBeforeRegularOrLikeExpression=expression operator_not? LIKE pattern=STRING_LITERAL (ESCAPE escapeSet=STRING_LITERAL)? | firstExpression=expression operator_not? operator_between secondExpression=expression operator_and thirdExpression=expression | unaryBeforeIsNullExpression=expression operator_is operator_not? null_literal | unaryBeforeInExpression=expression operator_not? (operator_in | operator_contains) LR_BRACKET constant (COMMA constant)* RR_BRACKET diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index dc9c12c7c8c5e..22cb1e0f539c3 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -129,6 +129,10 @@ CACHE : C A C H E ; +CALL + : C A L L + ; + CAST : C A S T ; @@ -149,6 +153,10 @@ CLUSTERID : C L U S T E R I D ; +COMMENT + : C O M M E N T + ; + CONCAT : C O N C A T ; @@ -161,6 +169,10 @@ CONFIGNODES : C O N F I G N O D E S ; +CONFIGNODE + : C O N F I G N O D E + ; + CONFIGURATION : C O N F I G U R A T I O N ; @@ -201,6 +213,14 @@ CREATE : C R E A T E ; +CURRENT_SQL_DIALECT + : C U R R E N T '_' S Q L '_' D I A L E C T + ; + +CURRENT_USER + : C U R R E N T '_' U S E R + ; + DATA : D A T A ; @@ -213,6 +233,10 @@ DATABASES : D A T A B A S E S ; +DATANODE + : D A T A N O D E + ; + DATANODEID : D A T A N O D E I D ; @@ -238,6 +262,10 @@ DELETE : D E L E T E ; +DEFAULT + : D E F A U L T + ; + DESC : D E S C ; @@ -282,6 +310,10 @@ ENDTIME : E N D T I M E ; +ESCAPE + : E S C A P E + ; + EVERY : E V E R Y ; @@ -294,6 +326,10 @@ EXPLAIN : E X P L A I N ; +EXTEND + : E X T E N D + ; + EXTRACTOR : E X T R A C T O R ; @@ -302,6 +338,10 @@ FALSE : F A L S E ; +FIELD + : F I E L D + ; + FILL : F I L L ; @@ -358,6 +398,10 @@ HAVING : H A V I N G ; +HEAD + : H E A D + ; + HYPERPARAMETERS : H Y P E R P A R A M E T E R S ; @@ -370,6 +414,10 @@ INDEX : I N D E X ; +INFERENCE + : I N F E R E N C E + ; + INFO : I N F O ; @@ -450,6 +498,18 @@ MIGRATE : M I G R A T E ; +AINODES + : A I N O D E S + ; + +MODEL + : M O D E L + ; + +MODELS + : M O D E L S + ; + MODIFY : M O D I F Y ; @@ -526,6 +586,10 @@ PASSWORD : P A S S W O R D ; +PATH + : P A T H + ; + PATHS : P A T H S ; @@ -614,6 +678,10 @@ READONLY : R E A D O N L Y ; +RECONSTRUCT + : R E C O N S T R U C T + ; + REGEXP : R E G E X P ; @@ -650,6 +718,10 @@ REPLACE : R E P L A C E ; +RESTRICT + : R E S T R I C T + ; + REVOKE : R E V O K E ; @@ -722,6 +794,10 @@ SPACE : S P A C E ; +SQL_DIALECT + : S Q L '_' D I A L E C T + ; + STORAGE : S T O R A G E ; @@ -750,6 +826,10 @@ STOP : S T O P ; +SUBSCRIPTION + : S U B S C R I P T I O N + ; + SUBSCRIPTIONS : S U B S C R I P T I O N S ; @@ -762,10 +842,22 @@ SYSTEM : S Y S T E M ; +TABLE + : T A B L E + ; + +TAG + : T A G + ; + TAGS : T A G S ; +TAIL + : T A I L + ; + TASK : T A S K ; @@ -830,6 +922,10 @@ TRACING : T R A C I N G ; +TREE + : T R E E + ; + TRIGGER : T R I G G E R ; diff --git a/iotdb-core/confignode/pom.xml b/iotdb-core/confignode/pom.xml index 168815c89dfff..2aee8f5210733 100644 --- a/iotdb-core/confignode/pom.xml +++ b/iotdb-core/confignode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT iotdb-confignode IoTDB: Core: ConfigNode @@ -42,57 +42,62 @@ org.apache.iotdb service-rpc - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-consensus - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-server - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb pipe-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb trigger-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb metrics-interface - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT + + + org.apache.iotdb + iotdb-thrift-ainode + 2.0.4-SNAPSHOT org.apache.iotdb node-commons - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.iotdb udf-api - 1.3.3-SNAPSHOT + 2.0.4-SNAPSHOT org.apache.tsfile @@ -149,6 +154,16 @@ mockito-core test + + com.tngtech.archunit + archunit + 1.3.0 + test + + + org.apache.commons + commons-math3 + diff --git a/iotdb-core/confignode/src/assembly/confignode.xml b/iotdb-core/confignode/src/assembly/confignode.xml index 638e47934752f..0c2e66d8f9582 100644 --- a/iotdb-core/confignode/src/assembly/confignode.xml +++ b/iotdb-core/confignode/src/assembly/confignode.xml @@ -37,8 +37,30 @@ conf - src/assembly/resources/sbin + ${project.basedir}/../../scripts/conf + conf + + confignode-env.* + **/confignode-env.* + + 0755 + + + ${project.basedir}/../../scripts/sbin sbin + + *confignode.* + **/*confignode.* + + 0755 + + + ${project.basedir}/../../scripts/tools + tools + + *confignode.* + **/*confignode.* + 0755 @@ -48,8 +70,8 @@ conf/iotdb-system.properties - ${project.basedir}/../node-commons/src/assembly/resources/sbin/iotdb-common.sh - sbin/iotdb-common.sh + ${project.basedir}/../../scripts/conf/iotdb-common.sh + conf/iotdb-common.sh diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/remove-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/remove-confignode.bat deleted file mode 100644 index d3e2cc143f5b5..0000000000000 --- a/iotdb-core/confignode/src/assembly/resources/sbin/remove-confignode.bat +++ /dev/null @@ -1,137 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM - -@echo off - -IF "%~1"=="--help" ( - echo The script will remove a ConfigNode. - echo Before removing a ConfigNode, ensure that there is at least one active ConfigNode in the cluster after the removal. - echo Usage: - echo Remove the ConfigNode with confignode_id - echo ./sbin/remove-confignode.bat [confignode_id] - echo Remove the ConfigNode with address:port - echo ./sbin/remove-confignode.bat [cn_internal_address:cn_internal_port] - EXIT /B 0 -) - -echo ``````````````````````````` -echo Starting to remove IoTDB ConfigNode -echo ``````````````````````````` - - -set PATH="%JAVA_HOME%\bin\";%PATH% -set "FULL_VERSION=" -set "MAJOR_VERSION=" -set "MINOR_VERSION=" - - -for /f tokens^=2-5^ delims^=.-_+^" %%j in ('java -fullversion 2^>^&1') do ( - set "FULL_VERSION=%%j-%%k-%%l-%%m" - IF "%%j" == "1" ( - set "MAJOR_VERSION=%%k" - set "MINOR_VERSION=%%l" - ) else ( - set "MAJOR_VERSION=%%j" - set "MINOR_VERSION=%%k" - ) -) - -set JAVA_VERSION=%MAJOR_VERSION% - -@REM we do not check jdk that version less than 1.8 because they are too stale... -IF "%JAVA_VERSION%" == "6" ( - echo IoTDB only supports jdk >= 8, please check your java version. - goto finally -) -IF "%JAVA_VERSION%" == "7" ( - echo IoTDB only supports jdk >= 8, please check your java version. - goto finally -) - -if "%OS%" == "Windows_NT" setlocal - -pushd %~dp0.. -if NOT DEFINED CONFIGNODE_HOME set CONFIGNODE_HOME=%cd% -popd - -set CONFIGNODE_CONF=%CONFIGNODE_HOME%\conf -set CONFIGNODE_LOGS=%CONFIGNODE_HOME%\logs - -@setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS -set CONF_PARAMS=-r -set is_conf_path=false -for %%i in (%*) do ( - IF "%%i" == "-c" ( - set is_conf_path=true - ) ELSE IF "!is_conf_path!" == "true" ( - set is_conf_path=false - set IOTDB_CONF=%%i - ) ELSE ( - set CONF_PARAMS=!CONF_PARAMS! %%i - ) -) - -IF EXIST "%CONFIGNODE_CONF%\confignode-env.bat" ( - CALL "%CONFIGNODE_CONF%\confignode-env.bat" %1 - ) ELSE ( - echo "can't find %CONFIGNODE_CONF%\confignode-env.bat" - ) - -if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.confignode.service.ConfigNode -if NOT DEFINED JAVA_HOME goto :err - -@REM ----------------------------------------------------------------------------- -@REM JVM Opts we'll use in legacy run or installation -set JAVA_OPTS=-ea^ - -Dlogback.configurationFile="%CONFIGNODE_CONF%\logback-confignode.xml"^ - -DCONFIGNODE_HOME="%CONFIGNODE_HOME%"^ - -DCONFIGNODE_CONF="%CONFIGNODE_CONF%"^ - -Dsun.jnu.encoding=UTF-8^ - -Dfile.encoding=UTF-8 - -@REM ***** CLASSPATH library setting ***** -@REM Ensure that any user defined CLASSPATH variables are not used on startup -if EXIST "%CONFIGNODE_HOME%\lib" (set CLASSPATH="%CONFIGNODE_HOME%\lib\*") else set CLASSPATH="%CONFIGNODE_HOME%\..\lib\*" -set CLASSPATH=%CLASSPATH%;iotdb.ConfigNode -goto okClasspath - -:append -set CLASSPATH=%CLASSPATH%;%1 - -goto :eof - -@REM ----------------------------------------------------------------------------- -:okClasspath - -rem echo CLASSPATH: %CLASSPATH% - -"%JAVA_HOME%\bin\java" %ILLEGAL_ACCESS_PARAMS% %JAVA_OPTS% %CONFIGNODE_HEAP_OPTS% -cp %CLASSPATH% %CONFIGNODE_JMX_OPTS% %MAIN_CLASS% %CONF_PARAMS% -goto finally - -:err -echo JAVA_HOME environment variable must be set! -pause - - -@REM ----------------------------------------------------------------------------- -:finally - -pause - -ENDLOCAL diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/remove-confignode.sh b/iotdb-core/confignode/src/assembly/resources/sbin/remove-confignode.sh deleted file mode 100755 index defa7a8eba09b..0000000000000 --- a/iotdb-core/confignode/src/assembly/resources/sbin/remove-confignode.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -if [ "$#" -eq 1 ] && [ "$1" == "--help" ]; then - echo "The script will remove a ConfigNode." - echo "Before removing a ConfigNode, ensure that there is at least one active ConfigNode in the cluster after the removal." - echo "Usage:" - echo "Remove the ConfigNode with confignode_id" - echo "./sbin/remove-confignode.sh [confignode_id]" - echo "Remove the ConfigNode with address:port" - echo "./sbin/remove-confignode.sh [cn_internal_address:cn_internal_port]" - exit 0 -fi - -echo ---------------------------- -echo Starting to remove IoTDB ConfigNode -echo ---------------------------- - -source "$(dirname "$0")/iotdb-common.sh" - -#get_iotdb_include wil remove -D parameters -VARS=$(get_iotdb_include "$*") -checkAllConfigNodeVariables -eval set -- "$VARS" - -PARAMS="-r "$* - -initConfigNodeEnv - -CLASSPATH="" -for f in ${CONFIGNODE_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done -classname=org.apache.iotdb.confignode.service.ConfigNode - -launch_service() { - class="$1" - iotdb_parms="-Dlogback.configurationFile=${CONFIGNODE_LOG_CONFIG}" - iotdb_parms="$iotdb_parms -DCONFIGNODE_HOME=${CONFIGNODE_HOME}" - iotdb_parms="$iotdb_parms -DCONFIGNODE_DATA_HOME=${CONFIGNODE_DATA_HOME}" - iotdb_parms="$iotdb_parms -DTSFILE_HOME=${CONFIGNODE_HOME}" - iotdb_parms="$iotdb_parms -DCONFIGNODE_CONF=${CONFIGNODE_CONF}" - iotdb_parms="$iotdb_parms -DTSFILE_CONF=${CONFIGNODE_CONF}" - iotdb_parms="$iotdb_parms -Dname=iotdb\.ConfigNode" - iotdb_parms="$iotdb_parms -DCONFIGNODE_LOGS=${CONFIGNODE_LOGS}" - - exec "$JAVA" $illegal_access_params $iotdb_parms $IOTDB_JMX_OPTS -cp "$CLASSPATH" "$class" $PARAMS - return $? -} - -# Start up the service -launch_service "$classname" - -exit $? diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat deleted file mode 100644 index 8c0e392f71bd0..0000000000000 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ /dev/null @@ -1,61 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM - -@echo off - -set current_dir=%~dp0 -set superior_dir=%current_dir%\..\ - -IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( - set config_file="%superior_dir%\conf\iotdb-system.properties" -) ELSE ( - IF EXIST "%superior_dir%\conf\iotdb-confignode.properties" ( - set config_file="%superior_dir%\conf\iotdb-confignode.properties" - ) ELSE ( - echo "No configuration file found. Exiting." - exit /b 1 - ) -) - -for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" -"%config_file%"') do ( - set cn_internal_port=%%i -) - -if not defined cn_internal_port ( - echo "cn_internal_port not found in the configuration file. Exiting." - exit /b 1 -) - -echo "check whether the cn_internal_port is used..., port is %cn_internal_port%" - -for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" -"%config_file%"') do ( - set cn_internal_address=%%i -) - -if not defined cn_internal_address ( - echo "cn_internal_address not found in the configuration file. Exiting." - exit /b 1 -) - -for /f "tokens=5" %%a in ('netstat /ano ^| findstr %cn_internal_address%:%cn_internal_port% ^| findstr LISTENING ') do ( - taskkill /f /pid %%a - echo "close ConfigNode, PID:" %%a -) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToCnNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToCnNodeRequestType.java index 3864f26cbc8e1..e422e45dff0be 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToCnNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToCnNodeRequestType.java @@ -27,7 +27,7 @@ public enum CnToCnNodeRequestType { REMOVE_CONFIG_NODE, DELETE_CONFIG_NODE_PEER, REPORT_CONFIG_NODE_SHUTDOWN, - STOP_CONFIG_NODE, + STOP_AND_CLEAR_CONFIG_NODE, SET_CONFIGURATION, SHOW_CONFIGURATION, SUBMIT_TEST_CONNECTION_TASK, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToDnRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToDnRequestType.java deleted file mode 100644 index bc73072825739..0000000000000 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/CnToDnRequestType.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.client; - -public enum CnToDnRequestType { - - // Node Maintenance - DISABLE_DATA_NODE, - STOP_DATA_NODE, - - FLUSH, - MERGE, - FULL_MERGE, - START_REPAIR_DATA, - STOP_REPAIR_DATA, - LOAD_CONFIGURATION, - SET_SYSTEM_STATUS, - SET_CONFIGURATION, - SHOW_CONFIGURATION, - - SUBMIT_TEST_CONNECTION_TASK, - TEST_CONNECTION, - - // Region Maintenance - CREATE_DATA_REGION, - CREATE_SCHEMA_REGION, - DELETE_REGION, - - CREATE_NEW_REGION_PEER, - ADD_REGION_PEER, - REMOVE_REGION_PEER, - DELETE_OLD_REGION_PEER, - RESET_PEER_LIST, - - UPDATE_REGION_ROUTE_MAP, - CHANGE_REGION_LEADER, - - // PartitionCache - INVALIDATE_PARTITION_CACHE, - INVALIDATE_PERMISSION_CACHE, - INVALIDATE_SCHEMA_CACHE, - CLEAR_CACHE, - - // Function - CREATE_FUNCTION, - DROP_FUNCTION, - - // Trigger - CREATE_TRIGGER_INSTANCE, - DROP_TRIGGER_INSTANCE, - ACTIVE_TRIGGER_INSTANCE, - INACTIVE_TRIGGER_INSTANCE, - UPDATE_TRIGGER_LOCATION, - - // Pipe Plugin - CREATE_PIPE_PLUGIN, - DROP_PIPE_PLUGIN, - - // Pipe Task - PIPE_PUSH_ALL_META, - PIPE_PUSH_SINGLE_META, - PIPE_PUSH_MULTI_META, - PIPE_HEARTBEAT, - - // Subscription - TOPIC_PUSH_ALL_META, - TOPIC_PUSH_SINGLE_META, - TOPIC_PUSH_MULTI_META, - CONSUMER_GROUP_PUSH_ALL_META, - CONSUMER_GROUP_PUSH_SINGLE_META, - - // CQ - EXECUTE_CQ, - - // TEMPLATE - UPDATE_TEMPLATE, - - // Schema - SET_TTL, - UPDATE_TTL_CACHE, - - CONSTRUCT_SCHEMA_BLACK_LIST, - ROLLBACK_SCHEMA_BLACK_LIST, - FETCH_SCHEMA_BLACK_LIST, - INVALIDATE_MATCHED_SCHEMA_CACHE, - DELETE_DATA_FOR_DELETE_SCHEMA, - DELETE_TIMESERIES, - - CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE, - ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE, - DEACTIVATE_TEMPLATE, - COUNT_PATHS_USING_TEMPLATE, - CHECK_SCHEMA_REGION_USING_TEMPLATE, - CHECK_TIMESERIES_EXISTENCE, - - CONSTRUCT_VIEW_SCHEMA_BLACK_LIST, - ROLLBACK_VIEW_SCHEMA_BLACK_LIST, - DELETE_VIEW, - - ALTER_VIEW, - - // TODO Need to migrate to Node Maintenance - KILL_QUERY_INSTANCE, - - // Quota - SET_SPACE_QUOTA, - SET_THROTTLE_QUOTA, -} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java new file mode 100644 index 0000000000000..e09ccc79becbf --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.async; + +import org.apache.iotdb.ainode.rpc.thrift.TAIHeartbeatReq; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.ClientPoolFactory; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.ainode.AsyncAINodeServiceClient; +import org.apache.iotdb.confignode.client.async.handlers.heartbeat.AINodeHeartbeatHandler; + +public class AsyncAINodeHeartbeatClientPool { + + private final IClientManager clientManager; + + private AsyncAINodeHeartbeatClientPool() { + clientManager = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.AsyncAINodeHeartbeatServiceClientPoolFactory()); + } + + public void getAINodeHeartBeat( + TEndPoint endPoint, TAIHeartbeatReq req, AINodeHeartbeatHandler handler) { + try { + clientManager.borrowClient(endPoint).getAIHeartbeat(req, handler); + } catch (Exception ignore) { + // Just ignore + } + } + + private static class AsyncAINodeHeartbeatClientPoolHolder { + + private static final AsyncAINodeHeartbeatClientPool INSTANCE = + new AsyncAINodeHeartbeatClientPool(); + + private AsyncAINodeHeartbeatClientPoolHolder() { + // Empty constructor + } + } + + public static AsyncAINodeHeartbeatClientPool getInstance() { + return AsyncAINodeHeartbeatClientPool.AsyncAINodeHeartbeatClientPoolHolder.INSTANCE; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java new file mode 100644 index 0000000000000..d138e333098d6 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.async; + +public enum CnToDnAsyncRequestType { + // Node Maintenance + STOP_AND_CLEAR_DATA_NODE, + CLEAN_DATA_NODE_CACHE, + FLUSH, + MERGE, + FULL_MERGE, + START_REPAIR_DATA, + STOP_REPAIR_DATA, + LOAD_CONFIGURATION, + SET_SYSTEM_STATUS, + SET_CONFIGURATION, + SUBMIT_TEST_CONNECTION_TASK, + SUBMIT_TEST_DN_INTERNAL_CONNECTION_TASK, + TEST_CONNECTION, + + // Region Maintenance + CREATE_DATA_REGION, + CREATE_SCHEMA_REGION, + DELETE_REGION, + RESET_PEER_LIST, + NOTIFY_REGION_MIGRATION, + UPDATE_REGION_ROUTE_MAP, + CHANGE_REGION_LEADER, + + // Cache + INVALIDATE_SCHEMA_CACHE, + INVALIDATE_LAST_CACHE, + CLEAR_CACHE, + + // Function + CREATE_FUNCTION, + DROP_FUNCTION, + + // Trigger + CREATE_TRIGGER_INSTANCE, + DROP_TRIGGER_INSTANCE, + ACTIVE_TRIGGER_INSTANCE, + INACTIVE_TRIGGER_INSTANCE, + UPDATE_TRIGGER_LOCATION, + + // Pipe Plugin + CREATE_PIPE_PLUGIN, + DROP_PIPE_PLUGIN, + + // Pipe Task + PIPE_PUSH_ALL_META, + PIPE_PUSH_SINGLE_META, + PIPE_PUSH_MULTI_META, + PIPE_HEARTBEAT, + + // Subscription + TOPIC_PUSH_ALL_META, + TOPIC_PUSH_SINGLE_META, + TOPIC_PUSH_MULTI_META, + CONSUMER_GROUP_PUSH_ALL_META, + CONSUMER_GROUP_PUSH_SINGLE_META, + + // TEMPLATE + UPDATE_TEMPLATE, + + // Schema + SET_TTL, + + CONSTRUCT_SCHEMA_BLACK_LIST, + ROLLBACK_SCHEMA_BLACK_LIST, + FETCH_SCHEMA_BLACK_LIST, + INVALIDATE_MATCHED_SCHEMA_CACHE, + DELETE_DATA_FOR_DELETE_SCHEMA, + DELETE_TIMESERIES, + + CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE, + ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE, + DEACTIVATE_TEMPLATE, + COUNT_PATHS_USING_TEMPLATE, + CHECK_SCHEMA_REGION_USING_TEMPLATE, + CHECK_TIMESERIES_EXISTENCE, + + CONSTRUCT_VIEW_SCHEMA_BLACK_LIST, + ROLLBACK_VIEW_SCHEMA_BLACK_LIST, + + DELETE_VIEW, + ALTER_VIEW, + + // TODO Need to migrate to Node Maintenance + KILL_QUERY_INSTANCE, + + // Quota + SET_SPACE_QUOTA, + SET_THROTTLE_QUOTA, + + // Table + UPDATE_TABLE, + INVALIDATE_TABLE_CACHE, + DELETE_DATA_FOR_DROP_TABLE, + DELETE_DEVICES_FOR_DROP_TABLE, + INVALIDATE_COLUMN_CACHE, + DELETE_COLUMN_DATA, + CONSTRUCT_TABLE_DEVICE_BLACK_LIST, + ROLLBACK_TABLE_DEVICE_BLACK_LIST, + INVALIDATE_MATCHED_TABLE_DEVICE_CACHE, + DELETE_DATA_FOR_TABLE_DEVICE, + DELETE_TABLE_DEVICE_IN_BLACK_LIST, + DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index 7ac3bbcd96168..3b7a5c4bfb7dd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -32,7 +32,7 @@ import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; import org.apache.iotdb.commons.client.request.DataNodeInternalServiceRequestManager; import org.apache.iotdb.commons.client.request.TestConnectionUtils; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.commons.exception.UncheckedStartupException; import org.apache.iotdb.confignode.client.async.handlers.rpc.CheckTimeSeriesExistenceRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.CountPathsUsingTemplateRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeAsyncRequestRPCHandler; @@ -43,6 +43,7 @@ import org.apache.iotdb.confignode.client.async.handlers.rpc.SchemaUpdateRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.SubmitTestConnectionTaskRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.TransferLeaderRPCHandler; +import org.apache.iotdb.confignode.client.async.handlers.rpc.TreeDeviceViewFieldDetectionHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.CheckSchemaRegionUsingTemplateRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.ConsumerGroupPushMetaRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler; @@ -50,6 +51,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; +import org.apache.iotdb.mpp.rpc.thrift.TCleanDataNodeCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListWithTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TConstructViewSchemaBlackListReq; @@ -60,15 +62,22 @@ import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteColumnDataReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteViewSchemaReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewReq; import org.apache.iotdb.mpp.rpc.thrift.TDropFunctionInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TDropPipePluginInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListReq; import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateColumnCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateTableCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq; import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq; @@ -84,279 +93,393 @@ import org.apache.iotdb.mpp.rpc.thrift.TRollbackSchemaBlackListReq; import org.apache.iotdb.mpp.rpc.thrift.TRollbackSchemaBlackListWithTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TRollbackViewSchemaBlackListReq; +import org.apache.iotdb.mpp.rpc.thrift.TTableDeviceDeletionWithPatternAndFilterReq; +import org.apache.iotdb.mpp.rpc.thrift.TTableDeviceDeletionWithPatternOrModReq; +import org.apache.iotdb.mpp.rpc.thrift.TTableDeviceInvalidateCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTriggerLocationReq; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + /** Asynchronously send RPC requests to DataNodes. See queryengine.thrift for more details. */ public class CnToDnInternalServiceAsyncRequestManager - extends DataNodeInternalServiceRequestManager { + extends DataNodeInternalServiceRequestManager { private static final Logger LOGGER = LoggerFactory.getLogger(CnToDnInternalServiceAsyncRequestManager.class); + @SuppressWarnings("unchecked") @Override protected void initActionMapBuilder() { actionMapBuilder.put( - CnToDnRequestType.SET_TTL, + CnToDnAsyncRequestType.SET_TTL, (req, client, handler) -> client.setTTL((TSetTTLReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CREATE_DATA_REGION, + CnToDnAsyncRequestType.CREATE_DATA_REGION, (req, client, handler) -> client.createDataRegion( (TCreateDataRegionReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DELETE_REGION, + CnToDnAsyncRequestType.DELETE_REGION, (req, client, handler) -> client.deleteRegion((TConsensusGroupId) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CREATE_SCHEMA_REGION, + CnToDnAsyncRequestType.CREATE_SCHEMA_REGION, (req, client, handler) -> client.createSchemaRegion( (TCreateSchemaRegionReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CREATE_FUNCTION, + CnToDnAsyncRequestType.CREATE_FUNCTION, (req, client, handler) -> client.createFunction( (TCreateFunctionInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DROP_FUNCTION, + CnToDnAsyncRequestType.DROP_FUNCTION, (req, client, handler) -> client.dropFunction( (TDropFunctionInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CREATE_TRIGGER_INSTANCE, + CnToDnAsyncRequestType.CREATE_TRIGGER_INSTANCE, (req, client, handler) -> client.createTriggerInstance( (TCreateTriggerInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DROP_TRIGGER_INSTANCE, + CnToDnAsyncRequestType.DROP_TRIGGER_INSTANCE, (req, client, handler) -> client.dropTriggerInstance( (TDropTriggerInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.ACTIVE_TRIGGER_INSTANCE, + CnToDnAsyncRequestType.ACTIVE_TRIGGER_INSTANCE, (req, client, handler) -> client.activeTriggerInstance( (TActiveTriggerInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.INACTIVE_TRIGGER_INSTANCE, + CnToDnAsyncRequestType.INACTIVE_TRIGGER_INSTANCE, (req, client, handler) -> client.inactiveTriggerInstance( (TInactiveTriggerInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.UPDATE_TRIGGER_LOCATION, + CnToDnAsyncRequestType.UPDATE_TRIGGER_LOCATION, (req, client, handler) -> client.updateTriggerLocation( (TUpdateTriggerLocationReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CREATE_PIPE_PLUGIN, + CnToDnAsyncRequestType.CREATE_PIPE_PLUGIN, (req, client, handler) -> client.createPipePlugin( (TCreatePipePluginInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DROP_PIPE_PLUGIN, + CnToDnAsyncRequestType.DROP_PIPE_PLUGIN, (req, client, handler) -> client.dropPipePlugin( (TDropPipePluginInstanceReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.PIPE_PUSH_ALL_META, + CnToDnAsyncRequestType.PIPE_PUSH_ALL_META, (req, client, handler) -> client.pushPipeMeta((TPushPipeMetaReq) req, (PipePushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.PIPE_PUSH_SINGLE_META, + CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, (req, client, handler) -> client.pushSinglePipeMeta( (TPushSinglePipeMetaReq) req, (PipePushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.PIPE_PUSH_MULTI_META, + CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, (req, client, handler) -> client.pushMultiPipeMeta( (TPushMultiPipeMetaReq) req, (PipePushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.TOPIC_PUSH_ALL_META, + CnToDnAsyncRequestType.TOPIC_PUSH_ALL_META, (req, client, handler) -> client.pushTopicMeta((TPushTopicMetaReq) req, (TopicPushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.TOPIC_PUSH_SINGLE_META, + CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, (req, client, handler) -> client.pushSingleTopicMeta( (TPushSingleTopicMetaReq) req, (TopicPushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.TOPIC_PUSH_MULTI_META, + CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, (req, client, handler) -> client.pushMultiTopicMeta( (TPushMultiTopicMetaReq) req, (TopicPushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CONSUMER_GROUP_PUSH_ALL_META, + CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_ALL_META, (req, client, handler) -> client.pushConsumerGroupMeta( (TPushConsumerGroupMetaReq) req, (ConsumerGroupPushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CONSUMER_GROUP_PUSH_SINGLE_META, + CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META, (req, client, handler) -> client.pushSingleConsumerGroupMeta( (TPushSingleConsumerGroupMetaReq) req, (ConsumerGroupPushMetaRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.PIPE_HEARTBEAT, + CnToDnAsyncRequestType.PIPE_HEARTBEAT, (req, client, handler) -> client.pipeHeartbeat((TPipeHeartbeatReq) req, (PipeHeartbeatRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.MERGE, + CnToDnAsyncRequestType.MERGE, (req, client, handler) -> client.merge((DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.FULL_MERGE, + CnToDnAsyncRequestType.FULL_MERGE, (req, client, handler) -> client.merge((DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.FLUSH, + CnToDnAsyncRequestType.FLUSH, (req, client, handler) -> client.flush((TFlushReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CLEAR_CACHE, - (req, client, handler) -> client.clearCache((DataNodeTSStatusRPCHandler) handler)); + CnToDnAsyncRequestType.CLEAR_CACHE, + (req, client, handler) -> + client.clearCache((Set) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.START_REPAIR_DATA, + CnToDnAsyncRequestType.START_REPAIR_DATA, (req, client, handler) -> client.startRepairData((DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.STOP_REPAIR_DATA, + CnToDnAsyncRequestType.STOP_REPAIR_DATA, (req, client, handler) -> client.stopRepairData((DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.LOAD_CONFIGURATION, + CnToDnAsyncRequestType.LOAD_CONFIGURATION, (req, client, handler) -> client.loadConfiguration((DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.SET_SYSTEM_STATUS, + CnToDnAsyncRequestType.SET_SYSTEM_STATUS, (req, client, handler) -> client.setSystemStatus((String) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.SET_CONFIGURATION, + CnToDnAsyncRequestType.SET_CONFIGURATION, (req, client, handler) -> client.setConfiguration( (TSetConfigurationReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.UPDATE_REGION_ROUTE_MAP, + CnToDnAsyncRequestType.UPDATE_REGION_ROUTE_MAP, (req, client, handler) -> client.updateRegionCache((TRegionRouteReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CHANGE_REGION_LEADER, + CnToDnAsyncRequestType.NOTIFY_REGION_MIGRATION, + (req, client, handler) -> + client.notifyRegionMigration( + (TNotifyRegionMigrationReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.CHANGE_REGION_LEADER, (req, client, handler) -> client.changeRegionLeader( (TRegionLeaderChangeReq) req, (TransferLeaderRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CONSTRUCT_SCHEMA_BLACK_LIST, + CnToDnAsyncRequestType.CONSTRUCT_SCHEMA_BLACK_LIST, (req, client, handler) -> client.constructSchemaBlackList( (TConstructSchemaBlackListReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.ROLLBACK_SCHEMA_BLACK_LIST, + CnToDnAsyncRequestType.ROLLBACK_SCHEMA_BLACK_LIST, (req, client, handler) -> client.rollbackSchemaBlackList( (TRollbackSchemaBlackListReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.FETCH_SCHEMA_BLACK_LIST, + CnToDnAsyncRequestType.FETCH_SCHEMA_BLACK_LIST, (req, client, handler) -> client.fetchSchemaBlackList( (TFetchSchemaBlackListReq) req, (FetchSchemaBlackListRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.INVALIDATE_MATCHED_SCHEMA_CACHE, + CnToDnAsyncRequestType.INVALIDATE_SCHEMA_CACHE, + (req, client, handler) -> + client.invalidateSchemaCache( + (TInvalidateCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.INVALIDATE_MATCHED_SCHEMA_CACHE, (req, client, handler) -> client.invalidateMatchedSchemaCache( (TInvalidateMatchedSchemaCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DELETE_DATA_FOR_DELETE_SCHEMA, + CnToDnAsyncRequestType.INVALIDATE_LAST_CACHE, + (req, client, handler) -> + client.invalidateLastCache((String) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_DATA_FOR_DELETE_SCHEMA, (req, client, handler) -> client.deleteDataForDeleteSchema( (TDeleteDataForDeleteSchemaReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DELETE_TIMESERIES, + CnToDnAsyncRequestType.DELETE_TIMESERIES, (req, client, handler) -> client.deleteTimeSeries((TDeleteTimeSeriesReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE, + CnToDnAsyncRequestType.CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE, (req, client, handler) -> client.constructSchemaBlackListWithTemplate( (TConstructSchemaBlackListWithTemplateReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE, + CnToDnAsyncRequestType.ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE, (req, client, handler) -> client.rollbackSchemaBlackListWithTemplate( (TRollbackSchemaBlackListWithTemplateReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DEACTIVATE_TEMPLATE, + CnToDnAsyncRequestType.DEACTIVATE_TEMPLATE, (req, client, handler) -> client.deactivateTemplate( (TDeactivateTemplateReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.UPDATE_TEMPLATE, + CnToDnAsyncRequestType.UPDATE_TEMPLATE, (req, client, handler) -> client.updateTemplate((TUpdateTemplateReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.COUNT_PATHS_USING_TEMPLATE, + CnToDnAsyncRequestType.COUNT_PATHS_USING_TEMPLATE, (req, client, handler) -> client.countPathsUsingTemplate( (TCountPathsUsingTemplateReq) req, (CountPathsUsingTemplateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CHECK_SCHEMA_REGION_USING_TEMPLATE, + CnToDnAsyncRequestType.CHECK_SCHEMA_REGION_USING_TEMPLATE, (req, client, handler) -> client.checkSchemaRegionUsingTemplate( (TCheckSchemaRegionUsingTemplateReq) req, (CheckSchemaRegionUsingTemplateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CHECK_TIMESERIES_EXISTENCE, + CnToDnAsyncRequestType.CHECK_TIMESERIES_EXISTENCE, (req, client, handler) -> client.checkTimeSeriesExistence( (TCheckTimeSeriesExistenceReq) req, (CheckTimeSeriesExistenceRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.CONSTRUCT_VIEW_SCHEMA_BLACK_LIST, + CnToDnAsyncRequestType.CONSTRUCT_VIEW_SCHEMA_BLACK_LIST, (req, client, handler) -> client.constructViewSchemaBlackList( (TConstructViewSchemaBlackListReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.ROLLBACK_VIEW_SCHEMA_BLACK_LIST, + CnToDnAsyncRequestType.ROLLBACK_VIEW_SCHEMA_BLACK_LIST, (req, client, handler) -> client.rollbackViewSchemaBlackList( (TRollbackViewSchemaBlackListReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.DELETE_VIEW, + CnToDnAsyncRequestType.DELETE_VIEW, (req, client, handler) -> client.deleteViewSchema((TDeleteViewSchemaReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.ALTER_VIEW, + CnToDnAsyncRequestType.ALTER_VIEW, (req, client, handler) -> client.alterView((TAlterViewReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.KILL_QUERY_INSTANCE, + CnToDnAsyncRequestType.KILL_QUERY_INSTANCE, (req, client, handler) -> client.killQueryInstance((String) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.SET_SPACE_QUOTA, + CnToDnAsyncRequestType.SET_SPACE_QUOTA, (req, client, handler) -> client.setSpaceQuota((TSetSpaceQuotaReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.SET_THROTTLE_QUOTA, + CnToDnAsyncRequestType.SET_THROTTLE_QUOTA, (req, client, handler) -> client.setThrottleQuota( (TSetThrottleQuotaReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.RESET_PEER_LIST, + CnToDnAsyncRequestType.RESET_PEER_LIST, (req, client, handler) -> client.resetPeerList((TResetPeerListReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.SUBMIT_TEST_CONNECTION_TASK, + CnToDnAsyncRequestType.SUBMIT_TEST_CONNECTION_TASK, (req, client, handler) -> client.submitTestConnectionTask( (TNodeLocations) req, (SubmitTestConnectionTaskRPCHandler) handler)); actionMapBuilder.put( - CnToDnRequestType.TEST_CONNECTION, + CnToDnAsyncRequestType.SUBMIT_TEST_DN_INTERNAL_CONNECTION_TASK, + (req, client, handler) -> + client.submitInternalTestConnectionTask( + (TNodeLocations) req, (SubmitTestConnectionTaskRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.TEST_CONNECTION, (req, client, handler) -> client.testConnectionEmptyRPC((DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.UPDATE_TABLE, + (req, client, handler) -> + client.updateTable((TUpdateTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.INVALIDATE_TABLE_CACHE, + (req, client, handler) -> + client.invalidateTableCache( + (TInvalidateTableCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, + (req, client, handler) -> + client.deleteDataForDropTable( + (TDeleteDataOrDevicesForDropTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, + (req, client, handler) -> + client.deleteDevicesForDropTable( + (TDeleteDataOrDevicesForDropTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.INVALIDATE_COLUMN_CACHE, + (req, client, handler) -> + client.invalidateColumnCache( + (TInvalidateColumnCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_COLUMN_DATA, + (req, client, handler) -> + client.deleteColumnData( + (TDeleteColumnDataReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.CONSTRUCT_TABLE_DEVICE_BLACK_LIST, + (req, client, handler) -> + client.constructTableDeviceBlackList( + (TTableDeviceDeletionWithPatternAndFilterReq) req, + (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.ROLLBACK_TABLE_DEVICE_BLACK_LIST, + (req, client, handler) -> + client.rollbackTableDeviceBlackList( + (TTableDeviceDeletionWithPatternOrModReq) req, + (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.INVALIDATE_MATCHED_TABLE_DEVICE_CACHE, + (req, client, handler) -> + client.invalidateMatchedTableDeviceCache( + (TTableDeviceInvalidateCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_DATA_FOR_TABLE_DEVICE, + (req, client, handler) -> + client.deleteDataForTableDevice( + (TTableDeviceDeletionWithPatternOrModReq) req, + (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_TABLE_DEVICE_IN_BLACK_LIST, + (req, client, handler) -> + client.deleteTableDeviceInBlackList( + (TTableDeviceDeletionWithPatternOrModReq) req, + (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, + (req, client, handler) -> + client.detectTreeDeviceViewFieldType( + (TDeviceViewReq) req, (TreeDeviceViewFieldDetectionHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE, + (req, client, handler) -> + client.cleanDataNodeCache( + (TCleanDataNodeCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.STOP_AND_CLEAR_DATA_NODE, + (req, client, handler) -> + client.stopAndClearDataNode((DataNodeTSStatusRPCHandler) handler)); + } + + @Override + protected void checkActionMapCompleteness() { + List lackList = + Arrays.stream(CnToDnAsyncRequestType.values()) + .filter(type -> !actionMap.containsKey(type)) + .collect(Collectors.toList()); + if (!lackList.isEmpty()) { + throw new UncheckedStartupException( + String.format("These request types should be added to actionMap: %s", lackList)); + } } @Override - protected AsyncRequestRPCHandler buildHandler( - AsyncRequestContext requestContext, + protected AsyncRequestRPCHandler buildHandler( + AsyncRequestContext requestContext, int requestId, TDataNodeLocation targetNode) { return DataNodeAsyncRequestRPCHandler.buildHandler(requestContext, requestId, targetNode); @@ -364,8 +487,8 @@ protected AsyncRequestRPCHandler buildH @Override protected void adjustClientTimeoutIfNecessary( - CnToDnRequestType cnToDnRequestType, AsyncDataNodeInternalServiceClient client) { - if (CnToDnRequestType.SUBMIT_TEST_CONNECTION_TASK.equals(cnToDnRequestType)) { + CnToDnAsyncRequestType CnToDnAsyncRequestType, AsyncDataNodeInternalServiceClient client) { + if (CnToDnAsyncRequestType.SUBMIT_TEST_CONNECTION_TASK.equals(CnToDnAsyncRequestType)) { client.setTimeoutTemporarily(TestConnectionUtils.calculateCnLeaderToAllDnMaxTime()); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/DataNodeAsyncRequestContext.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/DataNodeAsyncRequestContext.java index 2b813c081d7e7..50797d603a3b3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/DataNodeAsyncRequestContext.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/DataNodeAsyncRequestContext.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.client.request.AsyncRequestContext; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import java.util.Map; @@ -32,19 +32,21 @@ * @param ClassName of RPC response */ public class DataNodeAsyncRequestContext - extends AsyncRequestContext { + extends AsyncRequestContext { - public DataNodeAsyncRequestContext(CnToDnRequestType requestType) { + public DataNodeAsyncRequestContext(CnToDnAsyncRequestType requestType) { super(requestType); } public DataNodeAsyncRequestContext( - CnToDnRequestType requestType, Map dataNodeLocationMap) { + CnToDnAsyncRequestType requestType, Map dataNodeLocationMap) { super(requestType, dataNodeLocationMap); } public DataNodeAsyncRequestContext( - CnToDnRequestType requestType, Q q, Map dataNodeLocationMap) { + CnToDnAsyncRequestType requestType, + Q q, + Map dataNodeLocationMap) { super(requestType, q, dataNodeLocationMap); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java new file mode 100644 index 0000000000000..9d8e0b6e8474f --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.async.handlers.heartbeat; + +import org.apache.iotdb.ainode.rpc.thrift.TAIHeartbeatResp; +import org.apache.iotdb.commons.client.ThriftClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.cluster.NodeType; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; + +import org.apache.thrift.async.AsyncMethodCallback; + +public class AINodeHeartbeatHandler implements AsyncMethodCallback { + + private final int nodeId; + + private final LoadManager loadManager; + + public AINodeHeartbeatHandler(int nodeId, LoadManager loadManager) { + this.nodeId = nodeId; + this.loadManager = loadManager; + } + + @Override + public void onComplete(TAIHeartbeatResp aiHeartbeatResp) { + loadManager + .getLoadCache() + .cacheAINodeHeartbeatSample(nodeId, new NodeHeartbeatSample(aiHeartbeatResp)); + } + + @Override + public void onError(Exception e) { + if (ThriftClient.isConnectionBroken(e)) { + loadManager.forceUpdateNodeCache( + NodeType.DataNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown)); + } + loadManager.getLoadCache().resetHeartbeatProcessing(nodeId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java index 9d42c319a44e3..3613c337862aa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java @@ -87,10 +87,22 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) { .getLoadCache() .cacheDataNodeHeartbeatSample(nodeId, new NodeHeartbeatSample(heartbeatResp)); + RegionStatus regionStatus = RegionStatus.valueOf(heartbeatResp.getStatus()); + heartbeatResp .getJudgedLeaders() .forEach( (regionGroupId, isLeader) -> { + + // Do not allow regions to inherit the Removing state from datanode + RegionStatus nextRegionStatus = regionStatus; + if (nextRegionStatus == RegionStatus.Removing) { + nextRegionStatus = + loadManager + .getLoadCache() + .getRegionCacheLastSampleStatus(regionGroupId, nodeId); + } + // Update RegionGroupCache loadManager .getLoadCache() @@ -100,7 +112,7 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) { new RegionHeartbeatSample( heartbeatResp.getHeartbeatTimestamp(), // Region will inherit DataNode's status - RegionStatus.valueOf(heartbeatResp.getStatus())), + nextRegionStatus), false); if (((TConsensusGroupType.SchemaRegion.equals(regionGroupId.getType()) @@ -143,6 +155,9 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) { .updateConfirmedConfigNodeEndPoints( nodeId, heartbeatResp.getConfirmedConfigNodeEndPoints()); } + if (heartbeatResp.isSetRegionDisk()) { + loadManager.getLoadCache().updateRegionSizeMap(nodeId, heartbeatResp.getRegionDisk()); + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CheckTimeSeriesExistenceRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CheckTimeSeriesExistenceRPCHandler.java index b12ba289f1c2a..3a735691a0efa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CheckTimeSeriesExistenceRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CheckTimeSeriesExistenceRPCHandler.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -39,7 +39,7 @@ public class CheckTimeSeriesExistenceRPCHandler LoggerFactory.getLogger(CheckTimeSeriesExistenceRPCHandler.class); public CheckTimeSeriesExistenceRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CountPathsUsingTemplateRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CountPathsUsingTemplateRPCHandler.java index f2ed0cc425c57..b27c74bb41d8c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CountPathsUsingTemplateRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/CountPathsUsingTemplateRPCHandler.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -39,7 +39,7 @@ public class CountPathsUsingTemplateRPCHandler LoggerFactory.getLogger(CountPathsUsingTemplateRPCHandler.class); public CountPathsUsingTemplateRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java index f14a2365a9b35..85ce1253c13c9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java @@ -24,13 +24,14 @@ import org.apache.iotdb.common.rpc.thrift.TTestConnectionResp; import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.CheckSchemaRegionUsingTemplateRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.ConsumerGroupPushMetaRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateResp; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp; import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateResp; +import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewResp; import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListResp; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatResp; import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp; @@ -42,10 +43,10 @@ import java.util.concurrent.CountDownLatch; public abstract class DataNodeAsyncRequestRPCHandler - extends AsyncRequestRPCHandler { + extends AsyncRequestRPCHandler { protected DataNodeAsyncRequestRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetNode, Map dataNodeLocationMap, @@ -70,10 +71,10 @@ protected String generateFormattedTargetLocation(TDataNodeLocation dataNodeLocat } public static DataNodeAsyncRequestRPCHandler buildHandler( - AsyncRequestContext context, + AsyncRequestContext context, int requestId, TDataNodeLocation targetDataNode) { - CnToDnRequestType requestType = context.getRequestType(); + CnToDnAsyncRequestType requestType = context.getRequestType(); Map dataNodeLocationMap = context.getNodeLocationMap(); Map responseMap = context.getResponseMap(); CountDownLatch countDownLatch = context.getCountDownLatch(); @@ -174,6 +175,7 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( (Map) responseMap, countDownLatch); case SUBMIT_TEST_CONNECTION_TASK: + case SUBMIT_TEST_DN_INTERNAL_CONNECTION_TASK: return new SubmitTestConnectionTaskRPCHandler( requestType, requestId, @@ -181,6 +183,14 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( dataNodeLocationMap, (Map) responseMap, countDownLatch); + case DETECT_TREE_DEVICE_VIEW_FIELD_TYPE: + return new TreeDeviceViewFieldDetectionHandler( + requestType, + requestId, + targetDataNode, + dataNodeLocationMap, + (Map) responseMap, + countDownLatch); case SET_TTL: case CREATE_DATA_REGION: case CREATE_SCHEMA_REGION: @@ -195,16 +205,32 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( case FULL_MERGE: case FLUSH: case CLEAR_CACHE: + case INVALIDATE_LAST_CACHE: + case CLEAN_DATA_NODE_CACHE: + case STOP_AND_CLEAR_DATA_NODE: case START_REPAIR_DATA: case STOP_REPAIR_DATA: case LOAD_CONFIGURATION: case SET_SYSTEM_STATUS: + case NOTIFY_REGION_MIGRATION: case UPDATE_REGION_ROUTE_MAP: + case INVALIDATE_SCHEMA_CACHE: case INVALIDATE_MATCHED_SCHEMA_CACHE: case UPDATE_TEMPLATE: + case UPDATE_TABLE: case KILL_QUERY_INSTANCE: case RESET_PEER_LIST: case TEST_CONNECTION: + case INVALIDATE_TABLE_CACHE: + case DELETE_DATA_FOR_DROP_TABLE: + case DELETE_DEVICES_FOR_DROP_TABLE: + case INVALIDATE_COLUMN_CACHE: + case DELETE_COLUMN_DATA: + case CONSTRUCT_TABLE_DEVICE_BLACK_LIST: + case ROLLBACK_TABLE_DEVICE_BLACK_LIST: + case INVALIDATE_MATCHED_TABLE_DEVICE_CACHE: + case DELETE_DATA_FOR_TABLE_DEVICE: + case DELETE_TABLE_DEVICE_IN_BLACK_LIST: default: return new DataNodeTSStatusRPCHandler( requestType, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java index 19d451eb671fb..7c93f363dd4b8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -37,7 +37,7 @@ public class DataNodeTSStatusRPCHandler extends DataNodeAsyncRequestRPCHandler dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/FetchSchemaBlackListRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/FetchSchemaBlackListRPCHandler.java index 45c659298ca30..693017ec02d6a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/FetchSchemaBlackListRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/FetchSchemaBlackListRPCHandler.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -39,7 +39,7 @@ public class FetchSchemaBlackListRPCHandler LoggerFactory.getLogger(FetchSchemaBlackListRPCHandler.class); public FetchSchemaBlackListRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipeHeartbeatRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipeHeartbeatRPCHandler.java index ec4968cfa06dd..569424afbff43 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipeHeartbeatRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipeHeartbeatRPCHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.confignode.client.async.handlers.rpc; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatResp; import org.slf4j.Logger; @@ -34,7 +34,7 @@ public class PipeHeartbeatRPCHandler extends DataNodeAsyncRequestRPCHandler dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java index 3517bbeb93ea3..9ef80e9f8474a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.confignode.client.async.handlers.rpc; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -35,7 +35,7 @@ public class PipePushMetaRPCHandler extends DataNodeAsyncRequestRPCHandler dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SchemaUpdateRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SchemaUpdateRPCHandler.java index db8458a948ce3..dc2796a232e28 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SchemaUpdateRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SchemaUpdateRPCHandler.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -36,7 +36,7 @@ public class SchemaUpdateRPCHandler extends DataNodeTSStatusRPCHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SchemaUpdateRPCHandler.class); public SchemaUpdateRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SubmitTestConnectionTaskRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SubmitTestConnectionTaskRPCHandler.java index 4abf0a0eca114..f3c58892cbe4b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SubmitTestConnectionTaskRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/SubmitTestConnectionTaskRPCHandler.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TTestConnectionResp; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -38,7 +38,7 @@ public class SubmitTestConnectionTaskRPCHandler LoggerFactory.getLogger(SubmitTestConnectionTaskRPCHandler.class); public SubmitTestConnectionTaskRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TransferLeaderRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TransferLeaderRPCHandler.java index 352cc0694e2bf..8bfe0eb4755d3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TransferLeaderRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TransferLeaderRPCHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.confignode.client.async.handlers.rpc; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -37,7 +37,7 @@ public class TransferLeaderRPCHandler private static final Logger LOGGER = LoggerFactory.getLogger(TransferLeaderRPCHandler.class); public TransferLeaderRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TreeDeviceViewFieldDetectionHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TreeDeviceViewFieldDetectionHandler.java new file mode 100644 index 0000000000000..a0f908e94d45c --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/TreeDeviceViewFieldDetectionHandler.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.async.handlers.rpc; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewResp; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +public class TreeDeviceViewFieldDetectionHandler + extends DataNodeAsyncRequestRPCHandler { + + private static final Logger LOGGER = + LoggerFactory.getLogger(TreeDeviceViewFieldDetectionHandler.class); + + protected TreeDeviceViewFieldDetectionHandler( + final CnToDnAsyncRequestType requestType, + final int requestId, + final TDataNodeLocation targetNode, + final Map dataNodeLocationMap, + final Map integerTDeviceViewRespMap, + final CountDownLatch countDownLatch) { + super( + requestType, + requestId, + targetNode, + dataNodeLocationMap, + integerTDeviceViewRespMap, + countDownLatch); + } + + @Override + public void onComplete(final TDeviceViewResp response) { + // Put response + responseMap.put(requestId, response); + + if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.info("Successfully {} on DataNode: {}", requestType, formattedTargetLocation); + } else { + LOGGER.error( + "Failed to {} on DataNode: {}, response: {}", + requestType, + formattedTargetLocation, + response); + } + + // Always remove to avoid retrying + nodeLocationMap.remove(requestId); + + // Always CountDown + countDownLatch.countDown(); + } + + @Override + public void onError(final Exception e) { + final String errorMsg = + "Failed to " + + requestType + + " on DataNode: " + + formattedTargetLocation + + ", exception: " + + e.getMessage(); + LOGGER.warn(errorMsg, e); + + // Ensure that the map is always non-null + responseMap.put( + requestId, + new TDeviceViewResp() + .setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, errorMsg)) + .setDeviewViewFieldTypeMap(new HashMap<>())); + + // Always CountDown + countDownLatch.countDown(); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/CheckSchemaRegionUsingTemplateRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/CheckSchemaRegionUsingTemplateRPCHandler.java index 14898dcdc6c9c..249e8b5176799 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/CheckSchemaRegionUsingTemplateRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/CheckSchemaRegionUsingTemplateRPCHandler.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeAsyncRequestRPCHandler; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateResp; import org.apache.iotdb.rpc.RpcUtils; @@ -40,7 +40,7 @@ public class CheckSchemaRegionUsingTemplateRPCHandler LoggerFactory.getLogger(CheckSchemaRegionUsingTemplateRPCHandler.class); public CheckSchemaRegionUsingTemplateRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java index ee3c11eeb427b..2938d4f85b7cd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.confignode.client.async.handlers.rpc.subscription; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeAsyncRequestRPCHandler; import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp; import org.apache.iotdb.rpc.RpcUtils; @@ -38,7 +38,7 @@ public class ConsumerGroupPushMetaRPCHandler LoggerFactory.getLogger(ConsumerGroupPushMetaRPCHandler.class); public ConsumerGroupPushMetaRPCHandler( - CnToDnRequestType requestType, + CnToDnAsyncRequestType requestType, int requestId, TDataNodeLocation targetDataNode, Map dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java index cf8451feaacad..91ffdd7232b3f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.confignode.client.async.handlers.rpc.subscription; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeAsyncRequestRPCHandler; import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp; import org.apache.iotdb.rpc.RpcUtils; @@ -37,7 +37,7 @@ public class TopicPushMetaRPCHandler extends DataNodeAsyncRequestRPCHandler dataNodeLocationMap, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java new file mode 100644 index 0000000000000..21263612af258 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.sync; + +public enum CnToDnSyncRequestType { + // Node Maintenance + CLEAN_DATA_NODE_CACHE, + STOP_AND_CLEAR_DATA_NODE, + SET_SYSTEM_STATUS, + SHOW_CONFIGURATION, + + // Region Maintenance + CREATE_DATA_REGION, + CREATE_SCHEMA_REGION, + DELETE_REGION, + CREATE_NEW_REGION_PEER, + ADD_REGION_PEER, + REMOVE_REGION_PEER, + DELETE_OLD_REGION_PEER, + RESET_PEER_LIST, + + // PartitionCache + INVALIDATE_PARTITION_CACHE, + INVALIDATE_PERMISSION_CACHE, + INVALIDATE_SCHEMA_CACHE, + + // Template + UPDATE_TEMPLATE, + + // Schema + KILL_QUERY_INSTANCE, + + // Table + UPDATE_TABLE, +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java index cc26be000a8eb..6282924d0a0f1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java @@ -87,9 +87,9 @@ public Object sendSyncRequestToConfigNodeWithRetry( return client.deleteConfigNodePeer((TConfigNodeLocation) req); case REPORT_CONFIG_NODE_SHUTDOWN: return client.reportConfigNodeShutdown((TConfigNodeLocation) req); - case STOP_CONFIG_NODE: - // Only use stopConfigNode when the ConfigNode is removed. - return client.stopConfigNode((TConfigNodeLocation) req); + case STOP_AND_CLEAR_CONFIG_NODE: + // Only use stopAndClearConfigNode when the ConfigNode is removed. + return client.stopAndClearConfigNode((TConfigNodeLocation) req); case SET_CONFIGURATION: return client.setConfiguration((TSetConfigurationReq) req); case SHOW_CONFIGURATION: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java index 84e609ffb38c3..9a063900c0e6a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java @@ -27,50 +27,135 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; -import org.apache.iotdb.confignode.client.CnToDnRequestType; +import org.apache.iotdb.commons.exception.UncheckedStartupException; +import org.apache.iotdb.mpp.rpc.thrift.TCleanDataNodeCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq; import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq; -import org.apache.iotdb.mpp.rpc.thrift.TDisableDataNodeReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp; import org.apache.iotdb.mpp.rpc.thrift.TResetPeerListReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTemplateReq; -import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import com.google.common.collect.ImmutableMap; +import org.apache.ratis.util.function.CheckedBiFunction; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** Synchronously send RPC requests to DataNodes. See queryengine.thrift for more details. */ public class SyncDataNodeClientPool { private static final Logger LOGGER = LoggerFactory.getLogger(SyncDataNodeClientPool.class); - private static final int DEFAULT_RETRY_NUM = 6; + private static final int DEFAULT_RETRY_NUM = 10; private final IClientManager clientManager; + protected ImmutableMap< + CnToDnSyncRequestType, + CheckedBiFunction> + actionMap; + private SyncDataNodeClientPool() { clientManager = new IClientManager.Factory() .createClientManager( new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + buildActionMap(); + checkActionMapCompleteness(); + } + + private void buildActionMap() { + ImmutableMap.Builder< + CnToDnSyncRequestType, + CheckedBiFunction> + actionMapBuilder = ImmutableMap.builder(); + actionMapBuilder.put( + CnToDnSyncRequestType.INVALIDATE_PARTITION_CACHE, + (req, client) -> client.invalidatePartitionCache((TInvalidateCacheReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.INVALIDATE_SCHEMA_CACHE, + (req, client) -> client.invalidateSchemaCache((TInvalidateCacheReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.CREATE_SCHEMA_REGION, + (req, client) -> client.createSchemaRegion((TCreateSchemaRegionReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.CREATE_DATA_REGION, + (req, client) -> client.createDataRegion((TCreateDataRegionReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.DELETE_REGION, + (req, client) -> client.deleteRegion((TConsensusGroupId) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.INVALIDATE_PERMISSION_CACHE, + (req, client) -> client.invalidatePermissionCache((TInvalidatePermissionCacheReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.CLEAN_DATA_NODE_CACHE, + (req, client) -> client.cleanDataNodeCache((TCleanDataNodeCacheReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.STOP_AND_CLEAR_DATA_NODE, + (req, client) -> client.stopAndClearDataNode()); + actionMapBuilder.put( + CnToDnSyncRequestType.SET_SYSTEM_STATUS, + (req, client) -> client.setSystemStatus((String) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.KILL_QUERY_INSTANCE, + (req, client) -> client.killQueryInstance((String) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.UPDATE_TEMPLATE, + (req, client) -> client.updateTemplate((TUpdateTemplateReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.UPDATE_TABLE, + (req, client) -> client.updateTable((TUpdateTableReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.CREATE_NEW_REGION_PEER, + (req, client) -> client.createNewRegionPeer((TCreatePeerReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.ADD_REGION_PEER, + (req, client) -> client.addRegionPeer((TMaintainPeerReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.REMOVE_REGION_PEER, + (req, client) -> client.removeRegionPeer((TMaintainPeerReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.DELETE_OLD_REGION_PEER, + (req, client) -> client.deleteOldRegionPeer((TMaintainPeerReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.RESET_PEER_LIST, + (req, client) -> client.resetPeerList((TResetPeerListReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.SHOW_CONFIGURATION, (req, client) -> client.showConfiguration()); + actionMap = actionMapBuilder.build(); + } + + private void checkActionMapCompleteness() { + List lackList = + Arrays.stream(CnToDnSyncRequestType.values()) + .filter(type -> !actionMap.containsKey(type)) + .collect(Collectors.toList()); + if (!lackList.isEmpty()) { + throw new UncheckedStartupException( + String.format("These request types should be added to actionMap: %s", lackList)); + } } public Object sendSyncRequestToDataNodeWithRetry( - TEndPoint endPoint, Object req, CnToDnRequestType requestType) { + TEndPoint endPoint, Object req, CnToDnSyncRequestType requestType) { Throwable lastException = new TException(); for (int retry = 0; retry < DEFAULT_RETRY_NUM; retry++) { try (SyncDataNodeInternalServiceClient client = clientManager.borrowClient(endPoint)) { return executeSyncRequest(requestType, client, req); - } catch (ClientManagerException | TException e) { + } catch (Exception e) { lastException = e; if (retry != DEFAULT_RETRY_NUM - 1) { LOGGER.warn("{} failed on DataNode {}, retrying {}...", requestType, endPoint, retry + 1); @@ -84,12 +169,12 @@ public Object sendSyncRequestToDataNodeWithRetry( } public Object sendSyncRequestToDataNodeWithGivenRetry( - TEndPoint endPoint, Object req, CnToDnRequestType requestType, int retryNum) { + TEndPoint endPoint, Object req, CnToDnSyncRequestType requestType, int retryNum) { Throwable lastException = new TException(); for (int retry = 0; retry < retryNum; retry++) { try (SyncDataNodeInternalServiceClient client = clientManager.borrowClient(endPoint)) { return executeSyncRequest(requestType, client, req); - } catch (ClientManagerException | TException e) { + } catch (Exception e) { lastException = e; if (retry != retryNum - 1) { LOGGER.warn("{} failed on DataNode {}, retrying {}...", requestType, endPoint, retry + 1); @@ -103,47 +188,9 @@ public Object sendSyncRequestToDataNodeWithGivenRetry( } private Object executeSyncRequest( - CnToDnRequestType requestType, SyncDataNodeInternalServiceClient client, Object req) - throws TException { - switch (requestType) { - case INVALIDATE_PARTITION_CACHE: - return client.invalidatePartitionCache((TInvalidateCacheReq) req); - case INVALIDATE_SCHEMA_CACHE: - return client.invalidateSchemaCache((TInvalidateCacheReq) req); - case CREATE_SCHEMA_REGION: - return client.createSchemaRegion((TCreateSchemaRegionReq) req); - case CREATE_DATA_REGION: - return client.createDataRegion((TCreateDataRegionReq) req); - case DELETE_REGION: - return client.deleteRegion((TConsensusGroupId) req); - case INVALIDATE_PERMISSION_CACHE: - return client.invalidatePermissionCache((TInvalidatePermissionCacheReq) req); - case DISABLE_DATA_NODE: - return client.disableDataNode((TDisableDataNodeReq) req); - case STOP_DATA_NODE: - return client.stopDataNode(); - case SET_SYSTEM_STATUS: - return client.setSystemStatus((String) req); - case KILL_QUERY_INSTANCE: - return client.killQueryInstance((String) req); - case UPDATE_TEMPLATE: - return client.updateTemplate((TUpdateTemplateReq) req); - case CREATE_NEW_REGION_PEER: - return client.createNewRegionPeer((TCreatePeerReq) req); - case ADD_REGION_PEER: - return client.addRegionPeer((TMaintainPeerReq) req); - case REMOVE_REGION_PEER: - return client.removeRegionPeer((TMaintainPeerReq) req); - case DELETE_OLD_REGION_PEER: - return client.deleteOldRegionPeer((TMaintainPeerReq) req); - case RESET_PEER_LIST: - return client.resetPeerList((TResetPeerListReq) req); - case SHOW_CONFIGURATION: - return client.showConfiguration(); - default: - return RpcUtils.getStatus( - TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); - } + CnToDnSyncRequestType requestType, SyncDataNodeInternalServiceClient client, Object req) + throws Exception { + return Objects.requireNonNull(actionMap.get(requestType)).apply(req, client); } private void doRetryWait(int retryNum) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 1257e9246e948..50c8f5bc9a0d8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -20,14 +20,18 @@ package org.apache.iotdb.confignode.conf; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.confignode.manager.load.balancer.RegionBalancer; import org.apache.iotdb.confignode.manager.load.balancer.router.leader.AbstractLeaderBalancer; import org.apache.iotdb.confignode.manager.load.balancer.router.priority.IPriorityBalancer; +import org.apache.iotdb.confignode.manager.load.cache.IFailureDetector; import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import java.io.File; import java.lang.reflect.Field; @@ -35,7 +39,7 @@ public class ConfigNodeConfig { - /** ClusterId, the default value "defaultCluster" will be changed after join cluster. */ + /** ClusterName, the default value "defaultCluster" will be changed after join cluster. */ private volatile String clusterName = "defaultCluster"; /** ConfigNodeId, the default value -1 will be changed after join cluster. */ @@ -90,7 +94,7 @@ public class ConfigNodeConfig { private int defaultSchemaRegionGroupNumPerDatabase = 1; /** The maximum number of SchemaRegions expected to be managed by each DataNode. */ - private double schemaRegionPerDataNode = schemaReplicationFactor; + private int schemaRegionPerDataNode = 1; /** The policy of extension DataRegionGroup for each Database. */ private RegionGroupExtensionPolicy dataRegionGroupExtensionPolicy = @@ -103,15 +107,21 @@ public class ConfigNodeConfig { */ private int defaultDataRegionGroupNumPerDatabase = 2; - /** The maximum number of DataRegions expected to be managed by each DataNode. */ - private double dataRegionPerDataNode = 5.0; + /** + * The maximum number of DataRegions expected to be managed by each DataNode. Set to 0 means that + * each dataNode automatically has the number of CPU cores / 2 regions. + */ + private int dataRegionPerDataNode = 0; + + /** each dataNode automatically has the number of CPU cores / 2 regions. */ + private final double dataRegionPerDataNodeProportion = 0.5; /** RegionGroup allocate policy. */ private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy = RegionBalancer.RegionGroupAllocatePolicy.GCR; /** Max concurrent client number. */ - private int rpcMaxConcurrentClientNum = 65535; + private int rpcMaxConcurrentClientNum = 3000; /** just for test wait for 60 second by default. */ private int thriftServerAwaitTimeForStopService = 60; @@ -159,7 +169,7 @@ public class ConfigNodeConfig { systemDir + File.separator + "pipe" + File.separator + "receiver"; /** Procedure Evict ttl. */ - private int procedureCompletedEvictTTL = 800; + private int procedureCompletedEvictTTL = 60; /** Procedure completed clean interval. */ private int procedureCompletedCleanInterval = 30; @@ -171,8 +181,17 @@ public class ConfigNodeConfig { /** The heartbeat interval in milliseconds. */ private long heartbeatIntervalInMs = 1000; - /** The unknown DataNode detect interval in milliseconds. */ - private long unknownDataNodeDetectInterval = heartbeatIntervalInMs; + /** Failure detector implementation */ + private String failureDetector = IFailureDetector.PHI_ACCRUAL_DETECTOR; + + /** Max heartbeat elapsed time threshold for Fixed failure detector */ + private long failureDetectorFixedThresholdInMs = 20000; + + /** Max threshold for Phi accrual failure detector */ + private long failureDetectorPhiThreshold = 30; + + /** Acceptable pause duration for Phi accrual failure detector */ + private long failureDetectorPhiAcceptablePauseInMs = 10000; /** The policy of cluster RegionGroups' leader distribution. */ private String leaderDistributionPolicy = AbstractLeaderBalancer.CFD_POLICY; @@ -308,14 +327,10 @@ private void formulateFolders() { pipeReceiverFileDir = addHomeDir(pipeReceiverFileDir); } - private String addHomeDir(String dir) { - String homeDir = System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null); - if (!new File(dir).isAbsolute() && homeDir != null && homeDir.length() > 0) { - if (!homeDir.endsWith(File.separator)) { - dir = homeDir + File.separatorChar + dir; - } else { - dir = homeDir + dir; - } + public static String addHomeDir(String dir) { + final String homeDir = System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null); + if (!new File(dir).isAbsolute() && homeDir != null && !homeDir.isEmpty()) { + dir = !homeDir.endsWith(File.separator) ? homeDir + File.separatorChar + dir : homeDir + dir; } return dir; } @@ -339,6 +354,7 @@ public String getClusterName() { public void setClusterName(String clusterName) { this.clusterName = clusterName; + MetricConfigDescriptor.getInstance().getMetricConfig().updateClusterName(clusterName); } public int getConfigNodeId() { @@ -481,11 +497,11 @@ public void setDefaultDataRegionGroupNumPerDatabase(int defaultDataRegionGroupNu this.defaultDataRegionGroupNumPerDatabase = defaultDataRegionGroupNumPerDatabase; } - public double getSchemaRegionPerDataNode() { + public int getSchemaRegionPerDataNode() { return schemaRegionPerDataNode; } - public void setSchemaRegionPerDataNode(double schemaRegionPerDataNode) { + public void setSchemaRegionPerDataNode(int schemaRegionPerDataNode) { this.schemaRegionPerDataNode = schemaRegionPerDataNode; } @@ -497,14 +513,18 @@ public void setDataRegionConsensusProtocolClass(String dataRegionConsensusProtoc this.dataRegionConsensusProtocolClass = dataRegionConsensusProtocolClass; } - public double getDataRegionPerDataNode() { + public int getDataRegionPerDataNode() { return dataRegionPerDataNode; } - public void setDataRegionPerDataNode(double dataRegionPerDataNode) { + public void setDataRegionPerDataNode(int dataRegionPerDataNode) { this.dataRegionPerDataNode = dataRegionPerDataNode; } + public double getDataRegionPerDataNodeProportion() { + return dataRegionPerDataNodeProportion; + } + public RegionBalancer.RegionGroupAllocatePolicy getRegionGroupAllocatePolicy() { return regionGroupAllocatePolicy; } @@ -637,14 +657,6 @@ public void setHeartbeatIntervalInMs(long heartbeatIntervalInMs) { this.heartbeatIntervalInMs = heartbeatIntervalInMs; } - public long getUnknownDataNodeDetectInterval() { - return unknownDataNodeDetectInterval; - } - - public void setUnknownDataNodeDetectInterval(long unknownDataNodeDetectInterval) { - this.unknownDataNodeDetectInterval = unknownDataNodeDetectInterval; - } - public String getLeaderDistributionPolicy() { return leaderDistributionPolicy; } @@ -1199,4 +1211,43 @@ public TConfigNodeLocation generateLocalConfigNodeLocation() { new TEndPoint(getInternalAddress(), getInternalPort()), new TEndPoint(getInternalAddress(), getConsensusPort())); } + + public boolean isConsensusGroupStrongConsistency(TConsensusGroupId regionGroupId) { + return (TConsensusGroupType.SchemaRegion.equals(regionGroupId.getType()) + && getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)) + || (TConsensusGroupType.DataRegion.equals(regionGroupId.getType()) + && getDataRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)); + } + + public String getFailureDetector() { + return failureDetector; + } + + public void setFailureDetector(String failureDetector) { + this.failureDetector = failureDetector; + } + + public long getFailureDetectorFixedThresholdInMs() { + return failureDetectorFixedThresholdInMs; + } + + public void setFailureDetectorFixedThresholdInMs(long failureDetectorFixedThresholdInMs) { + this.failureDetectorFixedThresholdInMs = failureDetectorFixedThresholdInMs; + } + + public long getFailureDetectorPhiThreshold() { + return failureDetectorPhiThreshold; + } + + public void setFailureDetectorPhiThreshold(long failureDetectorPhiThreshold) { + this.failureDetectorPhiThreshold = failureDetectorPhiThreshold; + } + + public long getFailureDetectorPhiAcceptablePauseInMs() { + return failureDetectorPhiAcceptablePauseInMs; + } + + public void setFailureDetectorPhiAcceptablePauseInMs(long failureDetectorPhiAcceptablePauseInMs) { + this.failureDetectorPhiAcceptablePauseInMs = failureDetectorPhiAcceptablePauseInMs; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java index 5724eb1862fe9..17d23193b9d24 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java @@ -34,6 +34,7 @@ public class ConfigNodeConstant { "Executed failed, check usage: /:"; public static final String REMOVE_DATANODE_PROCESS = "[REMOVE_DATANODE_PROCESS]"; + public static final String REMOVE_AINODE_PROCESS = "[REMOVE_AINODE_PROCESS]"; public static final String REGION_MIGRATE_PROCESS = "[REGION_MIGRATE_PROCESS]"; private ConfigNodeConstant() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index 7719053587605..ed8c4bb5f2623 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -23,12 +23,14 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.conf.TrimProperties; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.apache.iotdb.confignode.manager.load.balancer.RegionBalancer; import org.apache.iotdb.confignode.manager.load.balancer.router.leader.AbstractLeaderBalancer; import org.apache.iotdb.confignode.manager.load.balancer.router.priority.IPriorityBalancer; +import org.apache.iotdb.confignode.manager.load.cache.IFailureDetector; import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.utils.NodeType; @@ -46,7 +48,6 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Optional; -import java.util.Properties; public class ConfigNodeDescriptor { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigNodeDescriptor.class); @@ -118,13 +119,13 @@ else if (!urlString.endsWith(".properties")) { } private void loadProps() { - Properties commonProperties = new Properties(); + TrimProperties trimProperties = new TrimProperties(); URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url != null) { try (InputStream inputStream = url.openStream()) { LOGGER.info("start reading ConfigNode conf file: {}", url); - commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - loadProperties(commonProperties); + trimProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + loadProperties(trimProperties); } catch (IOException | BadNodeUrlException e) { LOGGER.error("Couldn't load ConfigNode conf file, reject ConfigNode startup.", e); System.exit(-1); @@ -133,7 +134,7 @@ private void loadProps() { commonDescriptor .getConfig() .updatePath(System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null)); - MetricConfigDescriptor.getInstance().loadProps(commonProperties, true); + MetricConfigDescriptor.getInstance().loadProps(trimProperties, true); MetricConfigDescriptor.getInstance() .getMetricConfig() .updateRpcInstance(NodeType.CONFIGNODE, SchemaConstant.SYSTEM_DATABASE); @@ -145,27 +146,21 @@ private void loadProps() { } } - private void loadProperties(Properties properties) throws BadNodeUrlException, IOException { - conf.setClusterName( - properties.getProperty(IoTDBConstant.CLUSTER_NAME, conf.getClusterName()).trim()); + private void loadProperties(TrimProperties properties) throws BadNodeUrlException, IOException { + conf.setClusterName(properties.getProperty(IoTDBConstant.CLUSTER_NAME, conf.getClusterName())); conf.setInternalAddress( - properties - .getProperty(IoTDBConstant.CN_INTERNAL_ADDRESS, conf.getInternalAddress()) - .trim()); + properties.getProperty(IoTDBConstant.CN_INTERNAL_ADDRESS, conf.getInternalAddress())); conf.setInternalPort( Integer.parseInt( - properties - .getProperty(IoTDBConstant.CN_INTERNAL_PORT, String.valueOf(conf.getInternalPort())) - .trim())); + properties.getProperty( + IoTDBConstant.CN_INTERNAL_PORT, String.valueOf(conf.getInternalPort())))); conf.setConsensusPort( Integer.parseInt( - properties - .getProperty( - IoTDBConstant.CN_CONSENSUS_PORT, String.valueOf(conf.getConsensusPort())) - .trim())); + properties.getProperty( + IoTDBConstant.CN_CONSENSUS_PORT, String.valueOf(conf.getConsensusPort())))); String seedConfigNode = properties.getProperty(IoTDBConstant.CN_SEED_CONFIG_NODE, null); if (seedConfigNode == null) { @@ -176,150 +171,151 @@ private void loadProperties(Properties properties) throws BadNodeUrlException, I + "Please use cn_seed_config_node instead."); } if (seedConfigNode != null) { - conf.setSeedConfigNode(NodeUrlUtils.parseTEndPointUrls(seedConfigNode.trim()).get(0)); + conf.setSeedConfigNode(NodeUrlUtils.parseTEndPointUrls(seedConfigNode).get(0)); } conf.setSeriesSlotNum( Integer.parseInt( - properties - .getProperty("series_slot_num", String.valueOf(conf.getSeriesSlotNum())) - .trim())); + properties.getProperty("series_slot_num", String.valueOf(conf.getSeriesSlotNum())))); conf.setSeriesPartitionExecutorClass( - properties - .getProperty("series_partition_executor_class", conf.getSeriesPartitionExecutorClass()) - .trim()); + properties.getProperty( + "series_partition_executor_class", conf.getSeriesPartitionExecutorClass())); conf.setConfigNodeConsensusProtocolClass( - properties - .getProperty( - "config_node_consensus_protocol_class", conf.getConfigNodeConsensusProtocolClass()) - .trim()); + properties.getProperty( + "config_node_consensus_protocol_class", conf.getConfigNodeConsensusProtocolClass())); conf.setSchemaRegionConsensusProtocolClass( - properties - .getProperty( - "schema_region_consensus_protocol_class", - conf.getSchemaRegionConsensusProtocolClass()) - .trim()); + properties.getProperty( + "schema_region_consensus_protocol_class", + conf.getSchemaRegionConsensusProtocolClass())); conf.setSchemaReplicationFactor( Integer.parseInt( - properties - .getProperty( - "schema_replication_factor", String.valueOf(conf.getSchemaReplicationFactor())) - .trim())); + properties.getProperty( + "schema_replication_factor", String.valueOf(conf.getSchemaReplicationFactor())))); conf.setDataRegionConsensusProtocolClass( - properties - .getProperty( - "data_region_consensus_protocol_class", conf.getDataRegionConsensusProtocolClass()) - .trim()); + properties.getProperty( + "data_region_consensus_protocol_class", conf.getDataRegionConsensusProtocolClass())); conf.setDataReplicationFactor( Integer.parseInt( - properties - .getProperty( - "data_replication_factor", String.valueOf(conf.getDataReplicationFactor())) - .trim())); + properties.getProperty( + "data_replication_factor", String.valueOf(conf.getDataReplicationFactor())))); conf.setSchemaRegionGroupExtensionPolicy( RegionGroupExtensionPolicy.parse( properties.getProperty( "schema_region_group_extension_policy", - conf.getSchemaRegionGroupExtensionPolicy().getPolicy().trim()))); + conf.getSchemaRegionGroupExtensionPolicy().getPolicy()))); conf.setDefaultSchemaRegionGroupNumPerDatabase( Integer.parseInt( - properties - .getProperty( - "default_schema_region_group_num_per_database", - String.valueOf(conf.getDefaultSchemaRegionGroupNumPerDatabase())) - .trim())); + properties.getProperty( + "default_schema_region_group_num_per_database", + String.valueOf(conf.getDefaultSchemaRegionGroupNumPerDatabase())))); conf.setSchemaRegionPerDataNode( - Double.parseDouble( - properties - .getProperty( + (int) + Double.parseDouble( + properties.getProperty( "schema_region_per_data_node", - String.valueOf(conf.getSchemaRegionPerDataNode())) - .trim())); + String.valueOf(conf.getSchemaRegionPerDataNode())))); conf.setDataRegionGroupExtensionPolicy( RegionGroupExtensionPolicy.parse( properties.getProperty( "data_region_group_extension_policy", - conf.getDataRegionGroupExtensionPolicy().getPolicy().trim()))); + conf.getDataRegionGroupExtensionPolicy().getPolicy()))); conf.setDefaultDataRegionGroupNumPerDatabase( Integer.parseInt( properties.getProperty( "default_data_region_group_num_per_database", - String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase()).trim()))); + String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase())))); conf.setDataRegionPerDataNode( - Double.parseDouble( - properties - .getProperty( - "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode())) - .trim())); + (int) + Double.parseDouble( + properties.getProperty( + "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode())))); try { conf.setRegionAllocateStrategy( RegionBalancer.RegionGroupAllocatePolicy.valueOf( - properties - .getProperty( - "region_group_allocate_policy", conf.getRegionGroupAllocatePolicy().name()) - .trim())); + properties.getProperty( + "region_group_allocate_policy", conf.getRegionGroupAllocatePolicy().name()))); } catch (IllegalArgumentException e) { throw new IOException(e); } conf.setCnRpcMaxConcurrentClientNum( Integer.parseInt( - properties - .getProperty( - "cn_rpc_max_concurrent_client_num", - String.valueOf(conf.getCnRpcMaxConcurrentClientNum())) - .trim())); + properties.getProperty( + "cn_rpc_max_concurrent_client_num", + String.valueOf(conf.getCnRpcMaxConcurrentClientNum())))); conf.setMaxClientNumForEachNode( Integer.parseInt( - properties - .getProperty( - "cn_max_client_count_for_each_node_in_client_manager", - String.valueOf(conf.getMaxClientNumForEachNode())) - .trim())); + properties.getProperty( + "cn_max_client_count_for_each_node_in_client_manager", + String.valueOf(conf.getMaxClientNumForEachNode())))); - conf.setSystemDir(properties.getProperty("cn_system_dir", conf.getSystemDir()).trim()); + conf.setSystemDir(properties.getProperty("cn_system_dir", conf.getSystemDir())); - conf.setConsensusDir(properties.getProperty("cn_consensus_dir", conf.getConsensusDir()).trim()); + conf.setConsensusDir(properties.getProperty("cn_consensus_dir", conf.getConsensusDir())); - conf.setUdfDir(properties.getProperty("udf_lib_dir", conf.getUdfDir()).trim()); + conf.setUdfDir(properties.getProperty("udf_lib_dir", conf.getUdfDir())); - conf.setTriggerDir(properties.getProperty("trigger_lib_dir", conf.getTriggerDir()).trim()); + conf.setTriggerDir(properties.getProperty("trigger_lib_dir", conf.getTriggerDir())); - conf.setPipeDir(properties.getProperty("pipe_lib_dir", conf.getPipeDir()).trim()); + conf.setPipeDir(properties.getProperty("pipe_lib_dir", conf.getPipeDir())); conf.setPipeReceiverFileDir( Optional.ofNullable(properties.getProperty("cn_pipe_receiver_file_dir")) .orElse( properties.getProperty( "pipe_receiver_file_dir", - conf.getSystemDir() + File.separator + "pipe" + File.separator + "receiver")) - .trim()); + conf.getSystemDir() + File.separator + "pipe" + File.separator + "receiver"))); conf.setHeartbeatIntervalInMs( Long.parseLong( - properties - .getProperty( - "heartbeat_interval_in_ms", String.valueOf(conf.getHeartbeatIntervalInMs())) - .trim())); + properties.getProperty( + "heartbeat_interval_in_ms", String.valueOf(conf.getHeartbeatIntervalInMs())))); + + String failureDetector = properties.getProperty("failure_detector", conf.getFailureDetector()); + if (IFailureDetector.FIXED_DETECTOR.equals(failureDetector) + || IFailureDetector.PHI_ACCRUAL_DETECTOR.equals(failureDetector)) { + conf.setFailureDetector(failureDetector); + } else { + throw new IOException( + String.format( + "Unknown failure_detector: %s, " + "please set to \"fixed\" or \"phi_accrual\"", + failureDetector)); + } + + conf.setFailureDetectorFixedThresholdInMs( + Long.parseLong( + properties.getProperty( + "failure_detector_fixed_threshold_in_ms", + String.valueOf(conf.getFailureDetectorFixedThresholdInMs())))); + + conf.setFailureDetectorPhiThreshold( + Long.parseLong( + properties.getProperty( + "failure_detector_phi_threshold", + String.valueOf(conf.getFailureDetectorPhiThreshold())))); + + conf.setFailureDetectorPhiAcceptablePauseInMs( + Long.parseLong( + properties.getProperty( + "failure_detector_phi_acceptable_pause_in_ms", + String.valueOf(conf.getFailureDetectorPhiAcceptablePauseInMs())))); String leaderDistributionPolicy = - properties - .getProperty("leader_distribution_policy", conf.getLeaderDistributionPolicy()) - .trim(); + properties.getProperty("leader_distribution_policy", conf.getLeaderDistributionPolicy()); if (AbstractLeaderBalancer.GREEDY_POLICY.equals(leaderDistributionPolicy) || AbstractLeaderBalancer.CFD_POLICY.equals(leaderDistributionPolicy)) { conf.setLeaderDistributionPolicy(leaderDistributionPolicy); @@ -332,22 +328,18 @@ private void loadProperties(Properties properties) throws BadNodeUrlException, I conf.setEnableAutoLeaderBalanceForRatisConsensus( Boolean.parseBoolean( - properties - .getProperty( - "enable_auto_leader_balance_for_ratis_consensus", - String.valueOf(conf.isEnableAutoLeaderBalanceForRatisConsensus())) - .trim())); + properties.getProperty( + "enable_auto_leader_balance_for_ratis_consensus", + String.valueOf(conf.isEnableAutoLeaderBalanceForRatisConsensus())))); conf.setEnableAutoLeaderBalanceForIoTConsensus( Boolean.parseBoolean( - properties - .getProperty( - "enable_auto_leader_balance_for_iot_consensus", - String.valueOf(conf.isEnableAutoLeaderBalanceForIoTConsensus())) - .trim())); + properties.getProperty( + "enable_auto_leader_balance_for_iot_consensus", + String.valueOf(conf.isEnableAutoLeaderBalanceForIoTConsensus())))); String routePriorityPolicy = - properties.getProperty("route_priority_policy", conf.getRoutePriorityPolicy()).trim(); + properties.getProperty("route_priority_policy", conf.getRoutePriorityPolicy()); if (IPriorityBalancer.GREEDY_POLICY.equals(routePriorityPolicy) || IPriorityBalancer.LEADER_POLICY.equals(routePriorityPolicy)) { conf.setRoutePriorityPolicy(routePriorityPolicy); @@ -359,7 +351,7 @@ private void loadProperties(Properties properties) throws BadNodeUrlException, I } String readConsistencyLevel = - properties.getProperty("read_consistency_level", conf.getReadConsistencyLevel()).trim(); + properties.getProperty("read_consistency_level", conf.getReadConsistencyLevel()); if (readConsistencyLevel.equals("strong") || readConsistencyLevel.equals("weak")) { conf.setReadConsistencyLevel(readConsistencyLevel); } else { @@ -375,452 +367,343 @@ private void loadProperties(Properties properties) throws BadNodeUrlException, I conf.setProcedureCompletedEvictTTL( Integer.parseInt( - properties - .getProperty( - "procedure_completed_evict_ttl", - String.valueOf(conf.getProcedureCompletedEvictTTL())) - .trim())); + properties.getProperty( + "procedure_completed_evict_ttl", + String.valueOf(conf.getProcedureCompletedEvictTTL())))); conf.setProcedureCompletedCleanInterval( Integer.parseInt( - properties - .getProperty( - "procedure_completed_clean_interval", - String.valueOf(conf.getProcedureCompletedCleanInterval())) - .trim())); + properties.getProperty( + "procedure_completed_clean_interval", + String.valueOf(conf.getProcedureCompletedCleanInterval())))); conf.setProcedureCoreWorkerThreadsCount( Integer.parseInt( - properties - .getProperty( - "procedure_core_worker_thread_count", - String.valueOf(conf.getProcedureCoreWorkerThreadsCount())) - .trim())); + properties.getProperty( + "procedure_core_worker_thread_count", + String.valueOf(conf.getProcedureCoreWorkerThreadsCount())))); loadRatisConsensusConfig(properties); loadCQConfig(properties); } - private void loadRatisConsensusConfig(Properties properties) { + private void loadRatisConsensusConfig(TrimProperties properties) { conf.setDataRegionRatisConsensusLogAppenderBufferSize( Long.parseLong( - properties - .getProperty( - "data_region_ratis_log_appender_buffer_size_max", - String.valueOf(conf.getDataRegionRatisConsensusLogAppenderBufferSize())) - .trim())); + properties.getProperty( + "data_region_ratis_log_appender_buffer_size_max", + String.valueOf(conf.getDataRegionRatisConsensusLogAppenderBufferSize())))); conf.setConfigNodeRatisConsensusLogAppenderBufferSize( Long.parseLong( - properties - .getProperty( - "config_node_ratis_log_appender_buffer_size_max", - String.valueOf(conf.getConfigNodeRatisConsensusLogAppenderBufferSize())) - .trim())); + properties.getProperty( + "config_node_ratis_log_appender_buffer_size_max", + String.valueOf(conf.getConfigNodeRatisConsensusLogAppenderBufferSize())))); conf.setSchemaRegionRatisConsensusLogAppenderBufferSize( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_log_appender_buffer_size_max", - String.valueOf(conf.getSchemaRegionRatisConsensusLogAppenderBufferSize())) - .trim())); + properties.getProperty( + "schema_region_ratis_log_appender_buffer_size_max", + String.valueOf(conf.getSchemaRegionRatisConsensusLogAppenderBufferSize())))); conf.setDataRegionRatisSnapshotTriggerThreshold( Long.parseLong( - properties - .getProperty( - "data_region_ratis_snapshot_trigger_threshold", - String.valueOf(conf.getDataRegionRatisSnapshotTriggerThreshold())) - .trim())); + properties.getProperty( + "data_region_ratis_snapshot_trigger_threshold", + String.valueOf(conf.getDataRegionRatisSnapshotTriggerThreshold())))); conf.setConfigNodeRatisSnapshotTriggerThreshold( Long.parseLong( - properties - .getProperty( - "config_node_ratis_snapshot_trigger_threshold", - String.valueOf(conf.getConfigNodeRatisSnapshotTriggerThreshold())) - .trim())); + properties.getProperty( + "config_node_ratis_snapshot_trigger_threshold", + String.valueOf(conf.getConfigNodeRatisSnapshotTriggerThreshold())))); conf.setSchemaRegionRatisSnapshotTriggerThreshold( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_snapshot_trigger_threshold", - String.valueOf(conf.getSchemaRegionRatisSnapshotTriggerThreshold())) - .trim())); + properties.getProperty( + "schema_region_ratis_snapshot_trigger_threshold", + String.valueOf(conf.getSchemaRegionRatisSnapshotTriggerThreshold())))); conf.setDataRegionRatisLogUnsafeFlushEnable( Boolean.parseBoolean( - properties - .getProperty( - "data_region_ratis_log_unsafe_flush_enable", - String.valueOf(conf.isDataRegionRatisLogUnsafeFlushEnable())) - .trim())); + properties.getProperty( + "data_region_ratis_log_unsafe_flush_enable", + String.valueOf(conf.isDataRegionRatisLogUnsafeFlushEnable())))); conf.setConfigNodeRatisLogUnsafeFlushEnable( Boolean.parseBoolean( - properties - .getProperty( - "config_node_ratis_log_unsafe_flush_enable", - String.valueOf(conf.isConfigNodeRatisLogUnsafeFlushEnable())) - .trim())); + properties.getProperty( + "config_node_ratis_log_unsafe_flush_enable", + String.valueOf(conf.isConfigNodeRatisLogUnsafeFlushEnable())))); conf.setSchemaRegionRatisLogUnsafeFlushEnable( Boolean.parseBoolean( - properties - .getProperty( - "schema_region_ratis_log_unsafe_flush_enable", - String.valueOf(conf.isSchemaRegionRatisLogUnsafeFlushEnable())) - .trim())); + properties.getProperty( + "schema_region_ratis_log_unsafe_flush_enable", + String.valueOf(conf.isSchemaRegionRatisLogUnsafeFlushEnable())))); conf.setDataRegionRatisLogSegmentSizeMax( Long.parseLong( - properties - .getProperty( - "data_region_ratis_log_segment_size_max_in_byte", - String.valueOf(conf.getDataRegionRatisLogSegmentSizeMax())) - .trim())); + properties.getProperty( + "data_region_ratis_log_segment_size_max_in_byte", + String.valueOf(conf.getDataRegionRatisLogSegmentSizeMax())))); conf.setConfigNodeRatisLogSegmentSizeMax( Long.parseLong( - properties - .getProperty( - "config_node_ratis_log_segment_size_max_in_byte", - String.valueOf(conf.getConfigNodeRatisLogSegmentSizeMax())) - .trim())); + properties.getProperty( + "config_node_ratis_log_segment_size_max_in_byte", + String.valueOf(conf.getConfigNodeRatisLogSegmentSizeMax())))); conf.setSchemaRegionRatisLogSegmentSizeMax( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_log_segment_size_max_in_byte", - String.valueOf(conf.getSchemaRegionRatisLogSegmentSizeMax())) - .trim())); + properties.getProperty( + "schema_region_ratis_log_segment_size_max_in_byte", + String.valueOf(conf.getSchemaRegionRatisLogSegmentSizeMax())))); conf.setConfigNodeSimpleConsensusLogSegmentSizeMax( Long.parseLong( - properties - .getProperty( - "config_node_simple_consensus_log_segment_size_max_in_byte", - String.valueOf(conf.getConfigNodeSimpleConsensusLogSegmentSizeMax())) - .trim())); + properties.getProperty( + "config_node_simple_consensus_log_segment_size_max_in_byte", + String.valueOf(conf.getConfigNodeSimpleConsensusLogSegmentSizeMax())))); conf.setDataRegionRatisGrpcFlowControlWindow( Long.parseLong( - properties - .getProperty( - "data_region_ratis_grpc_flow_control_window", - String.valueOf(conf.getDataRegionRatisGrpcFlowControlWindow())) - .trim())); + properties.getProperty( + "data_region_ratis_grpc_flow_control_window", + String.valueOf(conf.getDataRegionRatisGrpcFlowControlWindow())))); conf.setConfigNodeRatisGrpcFlowControlWindow( Long.parseLong( - properties - .getProperty( - "config_node_ratis_grpc_flow_control_window", - String.valueOf(conf.getConfigNodeRatisGrpcFlowControlWindow())) - .trim())); + properties.getProperty( + "config_node_ratis_grpc_flow_control_window", + String.valueOf(conf.getConfigNodeRatisGrpcFlowControlWindow())))); conf.setSchemaRegionRatisGrpcFlowControlWindow( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_grpc_flow_control_window", - String.valueOf(conf.getSchemaRegionRatisGrpcFlowControlWindow())) - .trim())); + properties.getProperty( + "schema_region_ratis_grpc_flow_control_window", + String.valueOf(conf.getSchemaRegionRatisGrpcFlowControlWindow())))); conf.setDataRegionRatisGrpcLeaderOutstandingAppendsMax( Integer.parseInt( - properties - .getProperty( - "data_region_ratis_grpc_leader_outstanding_appends_max", - String.valueOf(conf.getDataRegionRatisGrpcLeaderOutstandingAppendsMax())) - .trim())); + properties.getProperty( + "data_region_ratis_grpc_leader_outstanding_appends_max", + String.valueOf(conf.getDataRegionRatisGrpcLeaderOutstandingAppendsMax())))); conf.setConfigNodeRatisGrpcLeaderOutstandingAppendsMax( Integer.parseInt( - properties - .getProperty( - "config_node_ratis_grpc_leader_outstanding_appends_max", - String.valueOf(conf.getConfigNodeRatisGrpcLeaderOutstandingAppendsMax())) - .trim())); + properties.getProperty( + "config_node_ratis_grpc_leader_outstanding_appends_max", + String.valueOf(conf.getConfigNodeRatisGrpcLeaderOutstandingAppendsMax())))); conf.setSchemaRegionRatisGrpcLeaderOutstandingAppendsMax( Integer.parseInt( - properties - .getProperty( - "schema_region_ratis_grpc_leader_outstanding_appends_max", - String.valueOf(conf.getSchemaRegionRatisGrpcLeaderOutstandingAppendsMax())) - .trim())); + properties.getProperty( + "schema_region_ratis_grpc_leader_outstanding_appends_max", + String.valueOf(conf.getSchemaRegionRatisGrpcLeaderOutstandingAppendsMax())))); conf.setDataRegionRatisLogForceSyncNum( Integer.parseInt( - properties - .getProperty( - "data_region_ratis_log_force_sync_num", - String.valueOf(conf.getDataRegionRatisLogForceSyncNum())) - .trim())); + properties.getProperty( + "data_region_ratis_log_force_sync_num", + String.valueOf(conf.getDataRegionRatisLogForceSyncNum())))); conf.setConfigNodeRatisLogForceSyncNum( Integer.parseInt( - properties - .getProperty( - "config_node_ratis_log_force_sync_num", - String.valueOf(conf.getConfigNodeRatisLogForceSyncNum())) - .trim())); + properties.getProperty( + "config_node_ratis_log_force_sync_num", + String.valueOf(conf.getConfigNodeRatisLogForceSyncNum())))); conf.setSchemaRegionRatisLogForceSyncNum( Integer.parseInt( - properties - .getProperty( - "schema_region_ratis_log_force_sync_num", - String.valueOf(conf.getSchemaRegionRatisLogForceSyncNum())) - .trim())); + properties.getProperty( + "schema_region_ratis_log_force_sync_num", + String.valueOf(conf.getSchemaRegionRatisLogForceSyncNum())))); conf.setDataRegionRatisRpcLeaderElectionTimeoutMinMs( Long.parseLong( - properties - .getProperty( - "data_region_ratis_rpc_leader_election_timeout_min_ms", - String.valueOf(conf.getDataRegionRatisRpcLeaderElectionTimeoutMinMs())) - .trim())); + properties.getProperty( + "data_region_ratis_rpc_leader_election_timeout_min_ms", + String.valueOf(conf.getDataRegionRatisRpcLeaderElectionTimeoutMinMs())))); conf.setConfigNodeRatisRpcLeaderElectionTimeoutMinMs( Long.parseLong( - properties - .getProperty( - "config_node_ratis_rpc_leader_election_timeout_min_ms", - String.valueOf(conf.getConfigNodeRatisRpcLeaderElectionTimeoutMinMs())) - .trim())); + properties.getProperty( + "config_node_ratis_rpc_leader_election_timeout_min_ms", + String.valueOf(conf.getConfigNodeRatisRpcLeaderElectionTimeoutMinMs())))); conf.setSchemaRegionRatisRpcLeaderElectionTimeoutMinMs( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_rpc_leader_election_timeout_min_ms", - String.valueOf(conf.getSchemaRegionRatisRpcLeaderElectionTimeoutMinMs())) - .trim())); + properties.getProperty( + "schema_region_ratis_rpc_leader_election_timeout_min_ms", + String.valueOf(conf.getSchemaRegionRatisRpcLeaderElectionTimeoutMinMs())))); conf.setDataRegionRatisRpcLeaderElectionTimeoutMaxMs( Long.parseLong( - properties - .getProperty( - "data_region_ratis_rpc_leader_election_timeout_max_ms", - String.valueOf(conf.getDataRegionRatisRpcLeaderElectionTimeoutMaxMs())) - .trim())); + properties.getProperty( + "data_region_ratis_rpc_leader_election_timeout_max_ms", + String.valueOf(conf.getDataRegionRatisRpcLeaderElectionTimeoutMaxMs())))); conf.setConfigNodeRatisRpcLeaderElectionTimeoutMaxMs( Long.parseLong( - properties - .getProperty( - "config_node_ratis_rpc_leader_election_timeout_max_ms", - String.valueOf(conf.getConfigNodeRatisRpcLeaderElectionTimeoutMaxMs())) - .trim())); + properties.getProperty( + "config_node_ratis_rpc_leader_election_timeout_max_ms", + String.valueOf(conf.getConfigNodeRatisRpcLeaderElectionTimeoutMaxMs())))); conf.setSchemaRegionRatisRpcLeaderElectionTimeoutMaxMs( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_rpc_leader_election_timeout_max_ms", - String.valueOf(conf.getSchemaRegionRatisRpcLeaderElectionTimeoutMaxMs())) - .trim())); + properties.getProperty( + "schema_region_ratis_rpc_leader_election_timeout_max_ms", + String.valueOf(conf.getSchemaRegionRatisRpcLeaderElectionTimeoutMaxMs())))); conf.setConfigNodeRatisRequestTimeoutMs( Long.parseLong( - properties - .getProperty( - "config_node_ratis_request_timeout_ms", - String.valueOf(conf.getConfigNodeRatisRequestTimeoutMs())) - .trim())); + properties.getProperty( + "config_node_ratis_request_timeout_ms", + String.valueOf(conf.getConfigNodeRatisRequestTimeoutMs())))); conf.setSchemaRegionRatisRequestTimeoutMs( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_request_timeout_ms", - String.valueOf(conf.getSchemaRegionRatisRequestTimeoutMs())) - .trim())); + properties.getProperty( + "schema_region_ratis_request_timeout_ms", + String.valueOf(conf.getSchemaRegionRatisRequestTimeoutMs())))); conf.setDataRegionRatisRequestTimeoutMs( Long.parseLong( - properties - .getProperty( - "data_region_ratis_request_timeout_ms", - String.valueOf(conf.getDataRegionRatisRequestTimeoutMs())) - .trim())); + properties.getProperty( + "data_region_ratis_request_timeout_ms", + String.valueOf(conf.getDataRegionRatisRequestTimeoutMs())))); conf.setConfigNodeRatisMaxRetryAttempts( Integer.parseInt( - properties - .getProperty( - "config_node_ratis_max_retry_attempts", - String.valueOf(conf.getConfigNodeRatisMaxRetryAttempts())) - .trim())); + properties.getProperty( + "config_node_ratis_max_retry_attempts", + String.valueOf(conf.getConfigNodeRatisMaxRetryAttempts())))); conf.setConfigNodeRatisInitialSleepTimeMs( Long.parseLong( - properties - .getProperty( - "config_node_ratis_initial_sleep_time_ms", - String.valueOf(conf.getConfigNodeRatisInitialSleepTimeMs())) - .trim())); + properties.getProperty( + "config_node_ratis_initial_sleep_time_ms", + String.valueOf(conf.getConfigNodeRatisInitialSleepTimeMs())))); conf.setConfigNodeRatisMaxSleepTimeMs( Long.parseLong( - properties - .getProperty( - "config_node_ratis_max_sleep_time_ms", - String.valueOf(conf.getConfigNodeRatisMaxSleepTimeMs())) - .trim())); + properties.getProperty( + "config_node_ratis_max_sleep_time_ms", + String.valueOf(conf.getConfigNodeRatisMaxSleepTimeMs())))); conf.setDataRegionRatisMaxRetryAttempts( Integer.parseInt( - properties - .getProperty( - "data_region_ratis_max_retry_attempts", - String.valueOf(conf.getDataRegionRatisMaxRetryAttempts())) - .trim())); + properties.getProperty( + "data_region_ratis_max_retry_attempts", + String.valueOf(conf.getDataRegionRatisMaxRetryAttempts())))); conf.setDataRegionRatisInitialSleepTimeMs( Long.parseLong( - properties - .getProperty( - "data_region_ratis_initial_sleep_time_ms", - String.valueOf(conf.getDataRegionRatisInitialSleepTimeMs())) - .trim())); + properties.getProperty( + "data_region_ratis_initial_sleep_time_ms", + String.valueOf(conf.getDataRegionRatisInitialSleepTimeMs())))); conf.setDataRegionRatisMaxSleepTimeMs( Long.parseLong( - properties - .getProperty( - "data_region_ratis_max_sleep_time_ms", - String.valueOf(conf.getDataRegionRatisMaxSleepTimeMs())) - .trim())); + properties.getProperty( + "data_region_ratis_max_sleep_time_ms", + String.valueOf(conf.getDataRegionRatisMaxSleepTimeMs())))); conf.setSchemaRegionRatisMaxRetryAttempts( Integer.parseInt( - properties - .getProperty( - "schema_region_ratis_max_retry_attempts", - String.valueOf(conf.getSchemaRegionRatisMaxRetryAttempts())) - .trim())); + properties.getProperty( + "schema_region_ratis_max_retry_attempts", + String.valueOf(conf.getSchemaRegionRatisMaxRetryAttempts())))); conf.setSchemaRegionRatisInitialSleepTimeMs( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_initial_sleep_time_ms", - String.valueOf(conf.getSchemaRegionRatisInitialSleepTimeMs())) - .trim())); + properties.getProperty( + "schema_region_ratis_initial_sleep_time_ms", + String.valueOf(conf.getSchemaRegionRatisInitialSleepTimeMs())))); conf.setSchemaRegionRatisMaxSleepTimeMs( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_max_sleep_time_ms", - String.valueOf(conf.getSchemaRegionRatisMaxSleepTimeMs())) - .trim())); + properties.getProperty( + "schema_region_ratis_max_sleep_time_ms", + String.valueOf(conf.getSchemaRegionRatisMaxSleepTimeMs())))); conf.setConfigNodeRatisPreserveLogsWhenPurge( Long.parseLong( - properties - .getProperty( - "config_node_ratis_preserve_logs_num_when_purge", - String.valueOf(conf.getConfigNodeRatisPreserveLogsWhenPurge())) - .trim())); + properties.getProperty( + "config_node_ratis_preserve_logs_num_when_purge", + String.valueOf(conf.getConfigNodeRatisPreserveLogsWhenPurge())))); conf.setSchemaRegionRatisPreserveLogsWhenPurge( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_preserve_logs_num_when_purge", - String.valueOf(conf.getSchemaRegionRatisPreserveLogsWhenPurge())) - .trim())); + properties.getProperty( + "schema_region_ratis_preserve_logs_num_when_purge", + String.valueOf(conf.getSchemaRegionRatisPreserveLogsWhenPurge())))); conf.setDataRegionRatisPreserveLogsWhenPurge( Long.parseLong( - properties - .getProperty( - "data_region_ratis_preserve_logs_num_when_purge", - String.valueOf(conf.getDataRegionRatisPreserveLogsWhenPurge())) - .trim())); + properties.getProperty( + "data_region_ratis_preserve_logs_num_when_purge", + String.valueOf(conf.getDataRegionRatisPreserveLogsWhenPurge())))); conf.setRatisFirstElectionTimeoutMinMs( Long.parseLong( - properties - .getProperty( - "ratis_first_election_timeout_min_ms", - String.valueOf(conf.getRatisFirstElectionTimeoutMinMs())) - .trim())); + properties.getProperty( + "ratis_first_election_timeout_min_ms", + String.valueOf(conf.getRatisFirstElectionTimeoutMinMs())))); conf.setRatisFirstElectionTimeoutMaxMs( Long.parseLong( - properties - .getProperty( - "ratis_first_election_timeout_max_ms", - String.valueOf(conf.getRatisFirstElectionTimeoutMaxMs())) - .trim())); + properties.getProperty( + "ratis_first_election_timeout_max_ms", + String.valueOf(conf.getRatisFirstElectionTimeoutMaxMs())))); conf.setConfigNodeRatisLogMax( Long.parseLong( - properties - .getProperty( - "config_node_ratis_log_max_size", - String.valueOf(conf.getConfigNodeRatisLogMax())) - .trim())); + properties.getProperty( + "config_node_ratis_log_max_size", + String.valueOf(conf.getConfigNodeRatisLogMax())))); conf.setSchemaRegionRatisLogMax( Long.parseLong( - properties - .getProperty( - "schema_region_ratis_log_max_size", - String.valueOf(conf.getSchemaRegionRatisLogMax())) - .trim())); + properties.getProperty( + "schema_region_ratis_log_max_size", + String.valueOf(conf.getSchemaRegionRatisLogMax())))); conf.setDataRegionRatisLogMax( Long.parseLong( - properties - .getProperty( - "data_region_ratis_log_max_size", - String.valueOf(conf.getDataRegionRatisLogMax())) - .trim())); + properties.getProperty( + "data_region_ratis_log_max_size", + String.valueOf(conf.getDataRegionRatisLogMax())))); conf.setConfigNodeRatisPeriodicSnapshotInterval( Long.parseLong( properties.getProperty( "config_node_ratis_periodic_snapshot_interval", - String.valueOf(conf.getConfigNodeRatisPeriodicSnapshotInterval()).trim()))); + String.valueOf(conf.getConfigNodeRatisPeriodicSnapshotInterval())))); conf.setSchemaRegionRatisPeriodicSnapshotInterval( Long.parseLong( properties.getProperty( "schema_region_ratis_periodic_snapshot_interval", - String.valueOf(conf.getSchemaRegionRatisPeriodicSnapshotInterval()).trim()))); + String.valueOf(conf.getSchemaRegionRatisPeriodicSnapshotInterval())))); conf.setDataRegionRatisPeriodicSnapshotInterval( Long.parseLong( properties.getProperty( "data_region_ratis_periodic_snapshot_interval", - String.valueOf(conf.getDataRegionRatisPeriodicSnapshotInterval()).trim()))); + String.valueOf(conf.getDataRegionRatisPeriodicSnapshotInterval())))); conf.setEnablePrintingNewlyCreatedPartition( Boolean.parseBoolean( - properties - .getProperty( - "enable_printing_newly_created_partition", - String.valueOf(conf.isEnablePrintingNewlyCreatedPartition())) - .trim())); + properties.getProperty( + "enable_printing_newly_created_partition", + String.valueOf(conf.isEnablePrintingNewlyCreatedPartition())))); conf.setForceWalPeriodForConfigNodeSimpleInMs( Long.parseLong( - properties - .getProperty( - "force_wal_period_for_confignode_simple_in_ms", - String.valueOf(conf.getForceWalPeriodForConfigNodeSimpleInMs())) - .trim())); + properties.getProperty( + "force_wal_period_for_confignode_simple_in_ms", + String.valueOf(conf.getForceWalPeriodForConfigNodeSimpleInMs())))); } - private void loadCQConfig(Properties properties) { + private void loadCQConfig(TrimProperties properties) { int cqSubmitThread = Integer.parseInt( - properties - .getProperty( - "continuous_query_submit_thread_count", - String.valueOf(conf.getCqSubmitThread())) - .trim()); + properties.getProperty( + "continuous_query_submit_thread_count", String.valueOf(conf.getCqSubmitThread()))); if (cqSubmitThread <= 0) { LOGGER.warn( "continuous_query_submit_thread should be greater than 0, " @@ -833,11 +716,9 @@ private void loadCQConfig(Properties properties) { long cqMinEveryIntervalInMs = Long.parseLong( - properties - .getProperty( - "continuous_query_min_every_interval_in_ms", - String.valueOf(conf.getCqMinEveryIntervalInMs())) - .trim()); + properties.getProperty( + "continuous_query_min_every_interval_in_ms", + String.valueOf(conf.getCqMinEveryIntervalInMs()))); if (cqMinEveryIntervalInMs <= 0) { LOGGER.warn( "continuous_query_min_every_interval_in_ms should be greater than 0, " @@ -871,7 +752,7 @@ public boolean isSeedConfigNode() { } } - public void loadHotModifiedProps(Properties properties) { + public void loadHotModifiedProps(TrimProperties properties) { Optional.ofNullable(properties.getProperty(IoTDBConstant.CLUSTER_NAME)) .ifPresent(conf::setClusterName); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeRemoveCheck.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeRemoveCheck.java index 9c4c480b02cea..b8475643ba25d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeRemoveCheck.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeRemoveCheck.java @@ -83,10 +83,7 @@ public TConfigNodeLocation removeCheck(String args) { .findFirst() .orElse(null); } catch (BadNodeUrlException e) { - LOGGER.info( - "Usage: remove-confignode.sh " - + "or remove-confignode.sh :", - e); + LOGGER.info("Use SQL: remove confignode ; ", e); return nodeLocation; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java index ced2964ec5cb5..7b599a02900de 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java @@ -25,6 +25,8 @@ import org.apache.iotdb.commons.exception.ConfigurationException; import org.apache.iotdb.commons.exception.StartupException; import org.apache.iotdb.commons.service.StartupChecks; +import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool; import org.apache.iotdb.confignode.manager.load.balancer.router.leader.AbstractLeaderBalancer; import org.apache.iotdb.confignode.manager.load.balancer.router.priority.IPriorityBalancer; import org.apache.iotdb.consensus.ConsensusFactory; @@ -73,6 +75,7 @@ public void startUpCheck() throws StartupException, IOException, ConfigurationEx verify(); checkGlobalConfig(); createDirsIfNecessary(); + checkRequestManager(); if (SystemPropertiesUtils.isRestarted()) { /* Always restore ConfigNodeId first */ CONF.setConfigNodeId(SystemPropertiesUtils.loadConfigNodeIdWhenRestarted()); @@ -142,16 +145,15 @@ private void checkGlobalConfig() throws ConfigurationException { "the SchemaRegion doesn't support org.apache.iotdb.consensus.iot.IoTConsensus"); } - // When the schemaengine region consensus protocol is set to PipeConsensus, + // When the schemaengine region consensus protocol is set to IoTConsensusV2, // we should report an error - if (CONF.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.FAST_IOT_CONSENSUS) - || CONF.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS_V2)) { + if (CONF.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS_V2)) { throw new ConfigurationException( "schema_region_consensus_protocol_class", String.valueOf(CONF.getSchemaRegionConsensusProtocolClass()), String.format( "%s or %s", ConsensusFactory.SIMPLE_CONSENSUS, ConsensusFactory.RATIS_CONSENSUS), - "the SchemaRegion doesn't support org.apache.iotdb.consensus.iot.FastIoTConsensus"); + "the SchemaRegion doesn't support org.apache.iotdb.consensus.iot.IoTConsensusV2"); } // The leader distribution policy is limited @@ -223,4 +225,10 @@ private void createDirIfEmpty(File dir) throws IOException { } } } + + // The checks are in the initialization process of the RequestManager object. + private void checkRequestManager() { + SyncDataNodeClientPool.getInstance(); + CnToDnInternalServiceAsyncRequestManager.getInstance(); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 8ad8c9b2fd25a..a3c6dc3b97b02 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -20,37 +20,15 @@ package org.apache.iotdb.confignode.consensus.request; import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException; -import org.apache.iotdb.confignode.consensus.request.auth.AuthorPlan; -import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; -import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; -import org.apache.iotdb.confignode.consensus.request.read.datanode.GetDataNodeConfigurationPlan; -import org.apache.iotdb.confignode.consensus.request.read.function.GetFunctionTablePlan; -import org.apache.iotdb.confignode.consensus.request.read.function.GetUDFJarPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.CountTimeSlotListPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetDataPartitionPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetNodePathsPartitionPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetOrCreateDataPartitionPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetOrCreateSchemaPartitionPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetSchemaPartitionPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetSeriesSlotListPlan; -import org.apache.iotdb.confignode.consensus.request.read.partition.GetTimeSlotListPlan; -import org.apache.iotdb.confignode.consensus.request.read.pipe.plugin.GetPipePluginJarPlan; -import org.apache.iotdb.confignode.consensus.request.read.pipe.plugin.GetPipePluginTablePlan; -import org.apache.iotdb.confignode.consensus.request.read.pipe.task.ShowPipePlanV2; -import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionIdPlan; -import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionInfoListPlan; -import org.apache.iotdb.confignode.consensus.request.read.subscription.ShowSubscriptionPlan; +import org.apache.iotdb.confignode.consensus.request.read.ainode.GetAINodeConfigurationPlan; +import org.apache.iotdb.confignode.consensus.request.read.model.GetModelInfoPlan; +import org.apache.iotdb.confignode.consensus.request.read.model.ShowModelPlan; import org.apache.iotdb.confignode.consensus.request.read.subscription.ShowTopicPlan; -import org.apache.iotdb.confignode.consensus.request.read.template.CheckTemplateSettablePlan; -import org.apache.iotdb.confignode.consensus.request.read.template.GetAllSchemaTemplatePlan; -import org.apache.iotdb.confignode.consensus.request.read.template.GetAllTemplateSetInfoPlan; -import org.apache.iotdb.confignode.consensus.request.read.template.GetPathsSetTemplatePlan; -import org.apache.iotdb.confignode.consensus.request.read.template.GetSchemaTemplatePlan; -import org.apache.iotdb.confignode.consensus.request.read.template.GetTemplateSetInfoPlan; -import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTransferringTriggersPlan; -import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTriggerJarPlan; -import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTriggerLocationPlan; -import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTriggerTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.UpdateAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan; @@ -58,7 +36,6 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; -import org.apache.iotdb.confignode.consensus.request.write.cq.ShowCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.UpdateCQLastExecTimePlan; import org.apache.iotdb.confignode.consensus.request.write.database.AdjustMaxRegionGroupNumPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; @@ -72,13 +49,22 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan; -import org.apache.iotdb.confignode.consensus.request.write.function.DropFunctionPlan; +import org.apache.iotdb.confignode.consensus.request.write.function.DropTableModelFunctionPlan; +import org.apache.iotdb.confignode.consensus.request.write.function.DropTreeModelFunctionPlan; +import org.apache.iotdb.confignode.consensus.request.write.function.UpdateFunctionPlan; +import org.apache.iotdb.confignode.consensus.request.write.model.CreateModelPlan; +import org.apache.iotdb.confignode.consensus.request.write.model.DropModelInNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan; +import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.AddRegionLocationPlan; +import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.UpdateRegionLocationPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteLogicalViewPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; @@ -114,7 +100,29 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.PreCreatePipePlanV1; import org.apache.iotdb.confignode.consensus.request.write.sync.RecordPipeMessagePlan; import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; -import org.apache.iotdb.confignode.consensus.request.write.sync.ShowPipePlanV1; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.SetTableColumnCommentPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.SetTableCommentPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.AddTableViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.CommitDeleteViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.CommitDeleteViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.PreCreateTableViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.PreDeleteViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.PreDeleteViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.RenameViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.RenameViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.SetViewCommentPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.SetViewPropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -132,8 +140,6 @@ import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.tsfile.utils.PublicBAOS; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.DataOutputStream; import java.io.IOException; @@ -142,11 +148,9 @@ public abstract class ConfigPhysicalPlan implements IConsensusRequest { - private static final Logger LOGGER = LoggerFactory.getLogger(ConfigPhysicalPlan.class); - private final ConfigPhysicalPlanType type; - protected ConfigPhysicalPlan(ConfigPhysicalPlanType type) { + protected ConfigPhysicalPlan(final ConfigPhysicalPlanType type) { this.type = type; } @@ -156,37 +160,37 @@ public ConfigPhysicalPlanType getType() { @Override public ByteBuffer serializeToByteBuffer() { - try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { serializeImpl(outputStream); return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); - } catch (IOException e) { + } catch (final IOException e) { throw new SerializationRunTimeException(e); } } - protected abstract void serializeImpl(DataOutputStream stream) throws IOException; + protected abstract void serializeImpl(final DataOutputStream stream) throws IOException; - protected abstract void deserializeImpl(ByteBuffer buffer) throws IOException; + protected abstract void deserializeImpl(final ByteBuffer buffer) throws IOException; public int getSerializedSize() throws IOException { - PublicBAOS byteArrayOutputStream = new PublicBAOS(); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); serializeImpl(outputStream); return byteArrayOutputStream.size(); } public static class Factory { - public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { - short planType = buffer.getShort(); - ConfigPhysicalPlanType configPhysicalPlanType = + public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOException { + final short planType = buffer.getShort(); + final ConfigPhysicalPlanType configPhysicalPlanType = ConfigPhysicalPlanType.convertToConfigPhysicalPlanType(planType); if (configPhysicalPlanType == null) { throw new IOException("Unrecognized log configPhysicalPlanType: " + planType); } - ConfigPhysicalPlan plan; + final ConfigPhysicalPlan plan; switch (configPhysicalPlanType) { case RegisterDataNode: plan = new RegisterDataNodePlan(); @@ -197,14 +201,23 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case RemoveDataNode: plan = new RemoveDataNodePlan(); break; - case GetDataNodeConfiguration: - plan = new GetDataNodeConfigurationPlan(); + case RegisterAINode: + plan = new RegisterAINodePlan(); + break; + case RemoveAINode: + plan = new RemoveAINodePlan(); + break; + case GetAINodeConfiguration: + plan = new GetAINodeConfigurationPlan(); + break; + case UpdateAINodeConfiguration: + plan = new UpdateAINodePlan(); break; case CreateDatabase: - plan = new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase); + plan = new DatabaseSchemaPlan(configPhysicalPlanType); break; case AlterDatabase: - plan = new DatabaseSchemaPlan(ConfigPhysicalPlanType.AlterDatabase); + plan = new DatabaseSchemaPlan(configPhysicalPlanType); break; case SetTTL: plan = new SetTTLPlan(); @@ -221,12 +234,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case AdjustMaxRegionGroupNum: plan = new AdjustMaxRegionGroupNumPlan(); break; - case CountDatabase: - plan = new CountDatabasePlan(); - break; - case GetDatabase: - plan = new GetDatabasePlan(); - break; case CreateRegionGroups: plan = new CreateRegionGroupsPlan(); break; @@ -248,23 +255,14 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case PollSpecificRegionMaintainTask: plan = new PollSpecificRegionMaintainTaskPlan(); break; - case GetSchemaPartition: - plan = new GetSchemaPartitionPlan(); - break; case CreateSchemaPartition: plan = new CreateSchemaPartitionPlan(); break; - case GetOrCreateSchemaPartition: - plan = new GetOrCreateSchemaPartitionPlan(); - break; - case GetDataPartition: - plan = new GetDataPartitionPlan(); - break; case CreateDataPartition: plan = new CreateDataPartitionPlan(); break; - case GetOrCreateDataPartition: - plan = new GetOrCreateDataPartitionPlan(); + case AutoCleanPartitionTable: + plan = new AutoCleanPartitionTablePlan(); break; case DeleteProcedure: plan = new DeleteProcedurePlan(); @@ -278,12 +276,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case DeleteDatabase: plan = new DeleteDatabasePlan(); break; - case ListUserDep: - case ListRoleDep: - case ListUserPrivilegeDep: - case ListRolePrivilegeDep: - case ListUserRolesDep: - case ListRoleUsersDep: case CreateUserDep: case CreateRoleDep: case DropUserDep: @@ -295,12 +287,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case RevokeRoleDep: case RevokeRoleFromUserDep: case UpdateUserDep: - case ListUser: - case ListRole: - case ListUserPrivilege: - case ListRolePrivilege: - case ListUserRoles: - case ListRoleUsers: case CreateUser: case CreateRole: case DropUser: @@ -313,7 +299,36 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case RevokeRoleFromUser: case UpdateUser: case CreateUserWithRawPassword: - plan = new AuthorPlan(configPhysicalPlanType); + plan = new AuthorTreePlan(configPhysicalPlanType); + break; + case RCreateUser: + case RCreateRole: + case RUpdateUser: + case RDropRole: + case RDropUser: + case RGrantUserRole: + case RRevokeUserRole: + case RGrantRoleAny: + case RGrantUserAny: + case RGrantUserAll: + case RGrantRoleAll: + case RGrantUserDBPriv: + case RGrantUserTBPriv: + case RGrantRoleDBPriv: + case RGrantRoleTBPriv: + case RRevokeRoleAny: + case RRevokeUserAny: + case RRevokeUserAll: + case RRevokeRoleAll: + case RRevokeUserDBPriv: + case RRevokeUserTBPriv: + case RRevokeRoleDBPriv: + case RRevokeRoleTBPriv: + case RGrantUserSysPri: + case RGrantRoleSysPri: + case RRevokeUserSysPri: + case RRevokeRoleSysPri: + plan = new AuthorRelationalPlan(configPhysicalPlanType); break; case ApplyConfigNode: plan = new ApplyConfigNodePlan(); @@ -330,8 +345,14 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case CreateFunction: plan = new CreateFunctionPlan(); break; - case DropFunction: - plan = new DropFunctionPlan(); + case UpdateFunction: + plan = new UpdateFunctionPlan(); + break; + case DropTreeModelFunction: + plan = new DropTreeModelFunctionPlan(); + break; + case DropTableModelFunction: + plan = new DropTableModelFunctionPlan(); break; case AddTriggerInTable: plan = new AddTriggerInTablePlan(); @@ -342,33 +363,9 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case UpdateTriggerStateInTable: plan = new UpdateTriggerStateInTablePlan(); break; - case GetTriggerTable: - plan = new GetTriggerTablePlan(); - break; - case GetTriggerLocation: - plan = new GetTriggerLocationPlan(); - break; - case GetTriggerJar: - plan = new GetTriggerJarPlan(); - break; case CreateSchemaTemplate: plan = new CreateSchemaTemplatePlan(); break; - case GetAllSchemaTemplate: - plan = new GetAllSchemaTemplatePlan(); - break; - case GetSchemaTemplate: - plan = new GetSchemaTemplatePlan(); - break; - case CheckTemplateSettable: - plan = new CheckTemplateSettablePlan(); - break; - case GetPathsSetTemplate: - plan = new GetPathsSetTemplatePlan(); - break; - case GetAllTemplateSetInfo: - plan = new GetAllTemplateSetInfoPlan(); - break; case SetSchemaTemplate: plan = new SetSchemaTemplatePlan(); break; @@ -378,9 +375,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case CommitSetSchemaTemplate: plan = new CommitSetSchemaTemplatePlan(); break; - case GetTemplateSetInfo: - plan = new GetTemplateSetInfoPlan(); - break; case DropSchemaTemplate: plan = new DropSchemaTemplatePlan(); break; @@ -396,11 +390,74 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case ExtendSchemaTemplate: plan = new ExtendSchemaTemplatePlan(); break; - case GetNodePathsPartition: - plan = new GetNodePathsPartitionPlan(); + case PreCreateTable: + plan = new PreCreateTablePlan(configPhysicalPlanType); + break; + case PreCreateTableView: + plan = new PreCreateTableViewPlan(); + break; + case RollbackCreateTable: + plan = new RollbackCreateTablePlan(); + break; + case CommitCreateTable: + plan = new CommitCreateTablePlan(); + break; + case AddTableColumn: + plan = new AddTableColumnPlan(configPhysicalPlanType); + break; + case AddViewColumn: + plan = new AddTableViewColumnPlan(); + break; + case SetTableProperties: + plan = new SetTablePropertiesPlan(configPhysicalPlanType); break; - case GetRegionInfoList: - plan = new GetRegionInfoListPlan(); + case SetViewProperties: + plan = new SetViewPropertiesPlan(); + break; + case RenameTableColumn: + plan = new RenameTableColumnPlan(configPhysicalPlanType); + break; + case RenameViewColumn: + plan = new RenameViewColumnPlan(); + break; + case PreDeleteTable: + plan = new PreDeleteTablePlan(configPhysicalPlanType); + break; + case PreDeleteView: + plan = new PreDeleteViewPlan(); + break; + case CommitDeleteTable: + plan = new CommitDeleteTablePlan(configPhysicalPlanType); + break; + case CommitDeleteView: + plan = new CommitDeleteViewPlan(); + break; + case PreDeleteColumn: + plan = new PreDeleteColumnPlan(configPhysicalPlanType); + break; + case PreDeleteViewColumn: + plan = new PreDeleteViewColumnPlan(); + break; + case CommitDeleteColumn: + plan = new CommitDeleteColumnPlan(configPhysicalPlanType); + break; + case CommitDeleteViewColumn: + plan = new CommitDeleteViewColumnPlan(); + break; + case SetTableComment: + plan = new SetTableCommentPlan(configPhysicalPlanType); + break; + case SetViewComment: + plan = new SetViewCommentPlan(); + break; + case SetTableColumnComment: + plan = new SetTableColumnCommentPlan(); + break; + case RenameTable: + plan = new RenameTablePlan(configPhysicalPlanType); + break; + case RenameView: + plan = new RenameViewPlan(); break; case CreatePipeSinkV1: plan = new CreatePipeSinkPlanV1(); @@ -420,9 +477,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case DropPipeV1: plan = new DropPipePlanV1(); break; - case ShowPipeV1: - plan = new ShowPipePlanV1(); - break; case RecordPipeMessageV1: plan = new RecordPipeMessagePlan(); break; @@ -438,9 +492,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case AlterPipeV2: plan = new AlterPipePlanV2(); break; - case ShowPipeV2: - plan = new ShowPipePlanV2(); - break; case OperateMultiplePipesV2: plan = new OperateMultiplePipesPlanV2(); break; @@ -477,9 +528,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case ConsumerGroupHandleMetaChange: plan = new ConsumerGroupHandleMetaChangePlan(); break; - case ShowSubscription: - plan = new ShowSubscriptionPlan(); - break; case PipeUnsetTemplate: plan = new PipeUnsetSchemaTemplatePlan(); break; @@ -492,17 +540,11 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case PipeDeactivateTemplate: plan = new PipeDeactivateTemplatePlan(); break; - case GetRegionId: - plan = new GetRegionIdPlan(); - break; - case GetTimeSlotList: - plan = new GetTimeSlotListPlan(); - break; - case CountTimeSlotList: - plan = new CountTimeSlotListPlan(); + case PipeCreateTableOrView: + plan = new PipeCreateTableOrViewPlan(); break; - case GetSeriesSlotList: - plan = new GetSeriesSlotListPlan(); + case PipeDeleteDevices: + plan = new PipeDeleteDevicesPlan(); break; case UpdateTriggersOnTransferNodes: plan = new UpdateTriggersOnTransferNodesPlan(); @@ -510,9 +552,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case UpdateTriggerLocation: plan = new UpdateTriggerLocationPlan(); break; - case GetTransferringTriggers: - plan = new GetTransferringTriggersPlan(); - break; case ACTIVE_CQ: plan = new ActiveCQPlan(); break; @@ -525,14 +564,23 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case UPDATE_CQ_LAST_EXEC_TIME: plan = new UpdateCQLastExecTimePlan(); break; - case SHOW_CQ: - plan = new ShowCQPlan(); + case CreateModel: + plan = new CreateModelPlan(); + break; + case UpdateModelInfo: + plan = new UpdateModelInfoPlan(); + break; + case DropModel: + plan = new DropModelPlan(); break; - case GetFunctionTable: - plan = new GetFunctionTablePlan(); + case ShowModel: + plan = new ShowModelPlan(); break; - case GetFunctionJar: - plan = new GetUDFJarPlan(); + case DropModelInNode: + plan = new DropModelInNodePlan(); + break; + case GetModelInfo: + plan = new GetModelInfoPlan(); break; case CreatePipePlugin: plan = new CreatePipePluginPlan(); @@ -540,12 +588,6 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case DropPipePlugin: plan = new DropPipePluginPlan(); break; - case GetPipePluginTable: - plan = new GetPipePluginTablePlan(); - break; - case GetPipePluginJar: - plan = new GetPipePluginJarPlan(); - break; case setSpaceQuota: plan = new SetSpaceQuotaPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 84728b3ca9b70..7cc542f7fa264 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -36,6 +36,12 @@ public enum ConfigPhysicalPlanType { RemoveDataNode((short) 102), UpdateDataNodeConfiguration((short) 103), + /** AINode. */ + RegisterAINode((short) 104), + UpdateAINodeConfiguration((short) 105), + RemoveAINode((short) 106), + GetAINodeConfiguration((short) 107), + /** Database. */ CreateDatabase((short) 200), SetTTL((short) 201), @@ -74,6 +80,7 @@ public enum ConfigPhysicalPlanType { CreateDataPartition((short) 404), GetOrCreateDataPartition((short) 405), GetNodePathsPartition((short) 406), + AutoCleanPartitionTable((short) 407), /** Procedure. */ UpdateProcedure((short) 500), @@ -125,11 +132,47 @@ public enum ConfigPhysicalPlanType { ListRoleUsers((short) 637), CreateUserWithRawPassword((short) 638), + /** Table Author */ + RCreateUser((short) 641), + RCreateRole((short) 642), + RUpdateUser((short) 643), + RDropUser((short) 644), + RDropRole((short) 645), + RGrantUserRole((short) 646), + RRevokeUserRole((short) 647), + RGrantUserAny((short) 648), + RGrantRoleAny((short) 649), + RGrantUserAll((short) 650), + RGrantRoleAll((short) 652), + RGrantUserDBPriv((short) 653), + RGrantUserTBPriv((short) 654), + RGrantRoleDBPriv((short) 655), + RGrantRoleTBPriv((short) 656), + RRevokeUserAny((short) 657), + RRevokeRoleAny((short) 658), + RRevokeUserAll((short) 659), + RRevokeRoleAll((short) 660), + RRevokeUserDBPriv((short) 661), + RRevokeUserTBPriv((short) 662), + RRevokeRoleDBPriv((short) 663), + RRevokeRoleTBPriv((short) 664), + RGrantUserSysPri((short) 665), + RGrantRoleSysPri((short) 666), + RRevokeUserSysPri((short) 667), + RRevokeRoleSysPri((short) 668), + RListUser((short) 669), + RListRole((short) 670), + RListUserPrivilege((short) 671), + RListRolePrivilege((short) 672), + /** Function. */ CreateFunction((short) 700), - DropFunction((short) 701), + DropTreeModelFunction((short) 701), GetFunctionTable((short) 702), GetFunctionJar((short) 703), + GetAllFunctionTable((short) 704), + UpdateFunction((short) 705), + DropTableModelFunction((short) 706), /** Template. */ CreateSchemaTemplate((short) 800), @@ -148,6 +191,36 @@ public enum ConfigPhysicalPlanType { CommitSetSchemaTemplate((short) 813), ExtendSchemaTemplate((short) 814), + /* Table or View */ + PreCreateTable((short) 850), + RollbackCreateTable((short) 851), + CommitCreateTable((short) 852), + AddTableColumn((short) 853), + SetTableProperties((short) 854), + ShowTable((short) 855), + FetchTable((short) 856), + RenameTableColumn((short) 857), + PreDeleteTable((short) 858), + CommitDeleteTable((short) 859), + PreDeleteColumn((short) 860), + CommitDeleteColumn((short) 861), + DescTable((short) 862), + ShowTable4InformationSchema((short) 863), + DescTable4InformationSchema((short) 864), + SetTableColumnComment((short) 865), + SetTableComment((short) 866), + RenameTable((short) 867), + PreCreateTableView((short) 868), + SetViewComment((short) 869), + AddViewColumn((short) 870), + CommitDeleteViewColumn((short) 871), + CommitDeleteView((short) 872), + RenameView((short) 873), + SetViewProperties((short) 874), + PreDeleteViewColumn((short) 875), + PreDeleteView((short) 876), + RenameViewColumn((short) 877), + /** Deprecated types for sync, restored them for upgrade. */ @Deprecated CreatePipeSinkV1((short) 900), @@ -184,7 +257,14 @@ public enum ConfigPhysicalPlanType { UPDATE_CQ_LAST_EXEC_TIME((short) 1103), SHOW_CQ((short) 1104), - // 1200-1299 planId is used by IoTDB-ML. + /** AI model. */ + CreateModel((short) 1200), + UpdateModelInfo((short) 1201), + UpdateModelState((short) 1202), + DropModel((short) 1203), + ShowModel((short) 1204), + GetModelInfo((short) 1206), + DropModelInNode((short) 1207), /** Pipe Plugin. */ CreatePipePlugin((short) 1300), @@ -215,6 +295,8 @@ public enum ConfigPhysicalPlanType { PipeDeleteLogicalView((short) 1703), PipeDeactivateTemplate((short) 1704), PipeSetTTL((short) 1705), + PipeCreateTableOrView((short) 1706), + PipeDeleteDevices((short) 1707), /** Subscription */ CreateTopic((short) 1800), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java index bfdf71c7e54be..774da4917fead 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java @@ -19,21 +19,39 @@ package org.apache.iotdb.confignode.consensus.request; -import org.apache.iotdb.confignode.consensus.request.auth.AuthorPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteLogicalViewPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeUnsetSchemaTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.SetTableColumnCommentPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.SetTableCommentPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.AddTableViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.CommitDeleteViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.CommitDeleteViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.RenameViewColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.RenameViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.SetViewCommentPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.view.SetViewPropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.ExtendSchemaTemplatePlan; public abstract class ConfigPhysicalPlanVisitor { - public R process(ConfigPhysicalPlan plan, C context) { + public R process(final ConfigPhysicalPlan plan, final C context) { switch (plan.getType()) { case CreateDatabase: return visitCreateDatabase((DatabaseSchemaPlan) plan, context); @@ -58,136 +76,428 @@ public R process(ConfigPhysicalPlan plan, C context) { case PipeDeactivateTemplate: return visitPipeDeactivateTemplate((PipeDeactivateTemplatePlan) plan, context); case CreateRole: - return visitCreateRole((AuthorPlan) plan, context); + return visitCreateRole((AuthorTreePlan) plan, context); case DropRole: - return visitDropRole((AuthorPlan) plan, context); + return visitDropRole((AuthorTreePlan) plan, context); case GrantRole: - return visitGrantRole((AuthorPlan) plan, context); + return visitGrantRole((AuthorTreePlan) plan, context); case RevokeRole: - return visitRevokeRole((AuthorPlan) plan, context); + return visitRevokeRole((AuthorTreePlan) plan, context); case CreateUser: - return visitCreateUser((AuthorPlan) plan, context); + return visitCreateUser((AuthorTreePlan) plan, context); case CreateUserWithRawPassword: - return visitCreateRawUser((AuthorPlan) plan, context); + return visitCreateRawUser((AuthorTreePlan) plan, context); case UpdateUser: - return visitUpdateUser((AuthorPlan) plan, context); + return visitUpdateUser((AuthorTreePlan) plan, context); case DropUser: - return visitDropUser((AuthorPlan) plan, context); + return visitDropUser((AuthorTreePlan) plan, context); case GrantUser: - return visitGrantUser((AuthorPlan) plan, context); + return visitGrantUser((AuthorTreePlan) plan, context); case RevokeUser: - return visitRevokeUser((AuthorPlan) plan, context); + return visitRevokeUser((AuthorTreePlan) plan, context); case GrantRoleToUser: - return visitGrantRoleToUser((AuthorPlan) plan, context); + return visitGrantRoleToUser((AuthorTreePlan) plan, context); case RevokeRoleFromUser: - return visitRevokeRoleFromUser((AuthorPlan) plan, context); + return visitRevokeRoleFromUser((AuthorTreePlan) plan, context); + case RCreateUser: + return visitRCreateUser((AuthorRelationalPlan) plan, context); + case RCreateRole: + return visitRCreateRole((AuthorRelationalPlan) plan, context); + case RUpdateUser: + return visitRUpdateUser((AuthorRelationalPlan) plan, context); + case RDropUser: + return visitRDropUserPlan((AuthorRelationalPlan) plan, context); + case RDropRole: + return visitRDropRolePlan((AuthorRelationalPlan) plan, context); + case RGrantUserRole: + return visitRGrantUserRole((AuthorRelationalPlan) plan, context); + case RRevokeUserRole: + return visitRRevokeUserRole((AuthorRelationalPlan) plan, context); + case RGrantUserAny: + return visitRGrantUserAny((AuthorRelationalPlan) plan, context); + case RGrantRoleAny: + return visitRGrantRoleAny((AuthorRelationalPlan) plan, context); + case RGrantUserAll: + return visitRGrantUserAll((AuthorRelationalPlan) plan, context); + case RGrantRoleAll: + return visitRGrantRoleAll((AuthorRelationalPlan) plan, context); + case RGrantUserDBPriv: + return visitRGrantUserDB((AuthorRelationalPlan) plan, context); + case RGrantUserTBPriv: + return visitRGrantUserTB((AuthorRelationalPlan) plan, context); + case RGrantRoleDBPriv: + return visitRGrantRoleDB((AuthorRelationalPlan) plan, context); + case RGrantRoleTBPriv: + return visitRGrantRoleTB((AuthorRelationalPlan) plan, context); + case RRevokeUserAny: + return visitRRevokeUserAny((AuthorRelationalPlan) plan, context); + case RRevokeRoleAny: + return visitRRevokeRoleAny((AuthorRelationalPlan) plan, context); + case RRevokeUserAll: + return visitRRevokeUserAll((AuthorRelationalPlan) plan, context); + case RRevokeRoleAll: + return visitRRevokeRoleAll((AuthorRelationalPlan) plan, context); + case RRevokeUserDBPriv: + return visitRRevokeUserDBPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeUserTBPriv: + return visitRRevokeUserTBPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeRoleDBPriv: + return visitRRevokeRoleDBPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeRoleTBPriv: + return visitRRevokeRoleTBPrivilege((AuthorRelationalPlan) plan, context); + case RGrantUserSysPri: + return visitRGrantUserSysPrivilege((AuthorRelationalPlan) plan, context); + case RGrantRoleSysPri: + return visitRGrantRoleSysPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeUserSysPri: + return visitRRevokeUserSysPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeRoleSysPri: + return visitRRevokeRoleSysPrivilege((AuthorRelationalPlan) plan, context); case SetTTL: return visitTTL((SetTTLPlan) plan, context); + case PipeCreateTableOrView: + return visitPipeCreateTableOrView((PipeCreateTableOrViewPlan) plan, context); + case AddTableColumn: + return visitAddTableColumn((AddTableColumnPlan) plan, context); + case AddViewColumn: + return visitAddTableViewColumn((AddTableViewColumnPlan) plan, context); + case SetTableProperties: + return visitSetTableProperties((SetTablePropertiesPlan) plan, context); + case SetViewProperties: + return visitSetViewProperties((SetViewPropertiesPlan) plan, context); + case RenameTableColumn: + return visitRenameTableColumn((RenameTableColumnPlan) plan, context); + case RenameViewColumn: + return visitRenameViewColumn((RenameViewColumnPlan) plan, context); + case CommitDeleteColumn: + return visitCommitDeleteColumn((CommitDeleteColumnPlan) plan, context); + case CommitDeleteViewColumn: + return visitCommitDeleteViewColumn((CommitDeleteViewColumnPlan) plan, context); + case CommitDeleteTable: + return visitCommitDeleteTable((CommitDeleteTablePlan) plan, context); + case CommitDeleteView: + return visitCommitDeleteView((CommitDeleteViewPlan) plan, context); + case PipeDeleteDevices: + return visitPipeDeleteDevices((PipeDeleteDevicesPlan) plan, context); + case SetTableComment: + return visitSetTableComment((SetTableCommentPlan) plan, context); + case SetViewComment: + return visitSetViewComment((SetViewCommentPlan) plan, context); + case SetTableColumnComment: + return visitSetTableColumnComment((SetTableColumnCommentPlan) plan, context); + case RenameTable: + return visitRenameTable((RenameTablePlan) plan, context); + case RenameView: + return visitRenameView((RenameViewPlan) plan, context); default: return visitPlan(plan, context); } } /** Top Level Description */ - public abstract R visitPlan(ConfigPhysicalPlan plan, C context); + public abstract R visitPlan(final ConfigPhysicalPlan plan, final C context); - public R visitCreateDatabase(DatabaseSchemaPlan createDatabasePlan, C context) { + public R visitCreateDatabase(final DatabaseSchemaPlan createDatabasePlan, final C context) { return visitPlan(createDatabasePlan, context); } - public R visitAlterDatabase(DatabaseSchemaPlan alterDatabasePlan, C context) { + public R visitAlterDatabase(final DatabaseSchemaPlan alterDatabasePlan, final C context) { return visitPlan(alterDatabasePlan, context); } - public R visitDeleteDatabase(DeleteDatabasePlan deleteDatabasePlan, C context) { + public R visitDeleteDatabase(final DeleteDatabasePlan deleteDatabasePlan, final C context) { return visitPlan(deleteDatabasePlan, context); } - public R visitCreateSchemaTemplate(CreateSchemaTemplatePlan createSchemaTemplatePlan, C context) { + public R visitCreateSchemaTemplate( + final CreateSchemaTemplatePlan createSchemaTemplatePlan, final C context) { return visitPlan(createSchemaTemplatePlan, context); } public R visitCommitSetSchemaTemplate( - CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan, C context) { + final CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan, final C context) { return visitPlan(commitSetSchemaTemplatePlan, context); } public R visitPipeUnsetSchemaTemplate( - PipeUnsetSchemaTemplatePlan pipeUnsetSchemaTemplatePlan, C context) { + final PipeUnsetSchemaTemplatePlan pipeUnsetSchemaTemplatePlan, final C context) { return visitPlan(pipeUnsetSchemaTemplatePlan, context); } - public R visitExtendSchemaTemplate(ExtendSchemaTemplatePlan extendSchemaTemplatePlan, C context) { + public R visitExtendSchemaTemplate( + final ExtendSchemaTemplatePlan extendSchemaTemplatePlan, final C context) { return visitPlan(extendSchemaTemplatePlan, context); } - public R visitDropSchemaTemplate(DropSchemaTemplatePlan dropSchemaTemplatePlan, C context) { + public R visitDropSchemaTemplate( + final DropSchemaTemplatePlan dropSchemaTemplatePlan, final C context) { return visitPlan(dropSchemaTemplatePlan, context); } - public R visitPipeDeleteTimeSeries(PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, C context) { + public R visitPipeDeleteTimeSeries( + final PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, final C context) { return visitPlan(pipeDeleteTimeSeriesPlan, context); } public R visitPipeDeleteLogicalView( - PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, C context) { + final PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, final C context) { return visitPlan(pipeDeleteLogicalViewPlan, context); } public R visitPipeDeactivateTemplate( - PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, C context) { + final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, final C context) { return visitPlan(pipeDeactivateTemplatePlan, context); } - public R visitCreateUser(AuthorPlan createUserPlan, C context) { + public R visitCreateUser(final AuthorTreePlan createUserPlan, final C context) { return visitPlan(createUserPlan, context); } - public R visitCreateRawUser(AuthorPlan createRawUserPlan, C context) { + public R visitCreateRawUser(final AuthorTreePlan createRawUserPlan, final C context) { return visitPlan(createRawUserPlan, context); } - public R visitUpdateUser(AuthorPlan updateUserPlan, C context) { + public R visitUpdateUser(final AuthorTreePlan updateUserPlan, final C context) { return visitPlan(updateUserPlan, context); } - public R visitDropUser(AuthorPlan dropUserPlan, C context) { + public R visitDropUser(final AuthorTreePlan dropUserPlan, final C context) { return visitPlan(dropUserPlan, context); } - public R visitGrantUser(AuthorPlan grantUserPlan, C context) { + public R visitGrantUser(final AuthorTreePlan grantUserPlan, final C context) { return visitPlan(grantUserPlan, context); } - public R visitRevokeUser(AuthorPlan revokeUserPlan, C context) { + public R visitRevokeUser(final AuthorTreePlan revokeUserPlan, final C context) { return visitPlan(revokeUserPlan, context); } - public R visitCreateRole(AuthorPlan createRolePlan, C context) { + public R visitCreateRole(final AuthorTreePlan createRolePlan, final C context) { return visitPlan(createRolePlan, context); } - public R visitDropRole(AuthorPlan dropRolePlan, C context) { + public R visitDropRole(final AuthorTreePlan dropRolePlan, final C context) { return visitPlan(dropRolePlan, context); } - public R visitGrantRole(AuthorPlan grantRolePlan, C context) { + public R visitGrantRole(final AuthorTreePlan grantRolePlan, final C context) { return visitPlan(grantRolePlan, context); } - public R visitRevokeRole(AuthorPlan revokeRolePlan, C context) { + public R visitRevokeRole(final AuthorTreePlan revokeRolePlan, final C context) { return visitPlan(revokeRolePlan, context); } - public R visitGrantRoleToUser(AuthorPlan grantRoleToUserPlan, C context) { + public R visitGrantRoleToUser(final AuthorTreePlan grantRoleToUserPlan, final C context) { return visitPlan(grantRoleToUserPlan, context); } - public R visitRevokeRoleFromUser(AuthorPlan revokeRoleFromUserPlan, C context) { + public R visitRevokeRoleFromUser(final AuthorTreePlan revokeRoleFromUserPlan, final C context) { return visitPlan(revokeRoleFromUserPlan, context); } - public R visitTTL(SetTTLPlan setTTLPlan, C context) { + public R visitRCreateUser(final AuthorRelationalPlan rCreateUserPlan, final C context) { + return visitPlan(rCreateUserPlan, context); + } + + public R visitRCreateRole(final AuthorRelationalPlan rCreateRolePlan, final C context) { + return visitPlan(rCreateRolePlan, context); + } + + public R visitRUpdateUser(final AuthorRelationalPlan rUpdateUserPlan, final C context) { + return visitPlan(rUpdateUserPlan, context); + } + + public R visitRDropUserPlan(final AuthorRelationalPlan rDropUserPlan, final C context) { + return visitPlan(rDropUserPlan, context); + } + + public R visitRDropRolePlan(final AuthorRelationalPlan rDropRolePlan, final C context) { + return visitPlan(rDropRolePlan, context); + } + + public R visitRGrantUserRole(final AuthorRelationalPlan rGrantUserRolePlan, final C context) { + return visitPlan(rGrantUserRolePlan, context); + } + + public R visitRRevokeUserRole(final AuthorRelationalPlan rRevokeUserRolePlan, final C context) { + return visitPlan(rRevokeUserRolePlan, context); + } + + public R visitRGrantUserAny(final AuthorRelationalPlan rGrantUserAnyPlan, final C context) { + return visitPlan(rGrantUserAnyPlan, context); + } + + public R visitRGrantRoleAny(final AuthorRelationalPlan rGrantRoleAnyPlan, final C context) { + return visitPlan(rGrantRoleAnyPlan, context); + } + + public R visitRGrantUserAll(final AuthorRelationalPlan rGrantUserAllPlan, final C context) { + return visitPlan(rGrantUserAllPlan, context); + } + + public R visitRGrantRoleAll(final AuthorRelationalPlan rGrantRoleAllPlan, final C context) { + return visitPlan(rGrantRoleAllPlan, context); + } + + public R visitRGrantUserDB(final AuthorRelationalPlan rGrantUserDBPlan, final C context) { + return visitPlan(rGrantUserDBPlan, context); + } + + public R visitRGrantUserTB(final AuthorRelationalPlan rGrantUserTBPlan, final C context) { + return visitPlan(rGrantUserTBPlan, context); + } + + public R visitRGrantRoleDB(final AuthorRelationalPlan rGrantRoleDBPlan, final C context) { + return visitPlan(rGrantRoleDBPlan, context); + } + + public R visitRGrantRoleTB(final AuthorRelationalPlan rGrantRoleTBPlan, final C context) { + return visitPlan(rGrantRoleTBPlan, context); + } + + public R visitRRevokeUserAny(final AuthorRelationalPlan rRevokeUserAnyPlan, final C context) { + return visitPlan(rRevokeUserAnyPlan, context); + } + + public R visitRRevokeRoleAny(final AuthorRelationalPlan rRevokeRoleAnyPlan, final C context) { + return visitPlan(rRevokeRoleAnyPlan, context); + } + + public R visitRRevokeUserAll(final AuthorRelationalPlan rRevokeUserAllPlan, final C context) { + return visitPlan(rRevokeUserAllPlan, context); + } + + public R visitRRevokeRoleAll(final AuthorRelationalPlan rRevokeRoleAllPlan, final C context) { + return visitPlan(rRevokeRoleAllPlan, context); + } + + public R visitRRevokeUserDBPrivilege( + final AuthorRelationalPlan rRevokeUserDBPrivilegePlan, final C context) { + return visitPlan(rRevokeUserDBPrivilegePlan, context); + } + + public R visitRRevokeUserTBPrivilege( + final AuthorRelationalPlan rRevokeUserTBPrivilegePlan, final C context) { + return visitPlan(rRevokeUserTBPrivilegePlan, context); + } + + public R visitRRevokeRoleDBPrivilege( + final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final C context) { + return visitPlan(rRevokeRoleTBPrivilegePlan, context); + } + + public R visitRRevokeRoleTBPrivilege( + final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final C context) { + return visitPlan(rRevokeRoleTBPrivilegePlan, context); + } + + public R visitRGrantUserSysPrivilege( + final AuthorRelationalPlan rGrantUserSysPrivilegePlan, final C context) { + return visitPlan(rGrantUserSysPrivilegePlan, context); + } + + public R visitRGrantRoleSysPrivilege( + final AuthorRelationalPlan rGrantRoleSysPrivilegePlan, final C context) { + return visitPlan(rGrantRoleSysPrivilegePlan, context); + } + + public R visitRRevokeUserSysPrivilege( + final AuthorRelationalPlan rRevokeUserSysPrivilegePlan, final C context) { + return visitPlan(rRevokeUserSysPrivilegePlan, context); + } + + public R visitRRevokeRoleSysPrivilege( + final AuthorRelationalPlan rRevokeRoleSysPrivilegePlan, final C context) { + return visitPlan(rRevokeRoleSysPrivilegePlan, context); + } + + public R visitTTL(final SetTTLPlan setTTLPlan, final C context) { return visitPlan(setTTLPlan, context); } + + public R visitPipeCreateTableOrView( + final PipeCreateTableOrViewPlan pipeCreateTableOrViewPlan, final C context) { + return visitPlan(pipeCreateTableOrViewPlan, context); + } + + public R visitAddTableColumn(final AddTableColumnPlan addTableColumnPlan, final C context) { + return visitPlan(addTableColumnPlan, context); + } + + // Use add table column by default + public R visitAddTableViewColumn( + final AddTableViewColumnPlan addTableViewColumnPlan, final C context) { + return visitAddTableColumn(addTableViewColumnPlan, context); + } + + public R visitSetTableProperties( + final SetTablePropertiesPlan setTablePropertiesPlan, final C context) { + return visitPlan(setTablePropertiesPlan, context); + } + + // Use set table properties by default + public R visitSetViewProperties( + final SetViewPropertiesPlan setViewPropertiesPlan, final C context) { + return visitSetTableProperties(setViewPropertiesPlan, context); + } + + public R visitCommitDeleteColumn( + final CommitDeleteColumnPlan commitDeleteColumnPlan, final C context) { + return visitPlan(commitDeleteColumnPlan, context); + } + + public R visitCommitDeleteViewColumn( + final CommitDeleteViewColumnPlan commitDeleteViewColumnPlan, final C context) { + return visitCommitDeleteColumn(commitDeleteViewColumnPlan, context); + } + + public R visitRenameTableColumn( + final RenameTableColumnPlan renameTableColumnPlan, final C context) { + return visitPlan(renameTableColumnPlan, context); + } + + // Use commit delete table by default + public R visitRenameViewColumn(final RenameViewColumnPlan renameViewColumnPlan, final C context) { + return visitRenameTableColumn(renameViewColumnPlan, context); + } + + public R visitCommitDeleteTable( + final CommitDeleteTablePlan commitDeleteTablePlan, final C context) { + return visitPlan(commitDeleteTablePlan, context); + } + + // Use commit delete table by default + public R visitCommitDeleteView(final CommitDeleteViewPlan commitDeleteViewPlan, final C context) { + return visitCommitDeleteTable(commitDeleteViewPlan, context); + } + + public R visitPipeDeleteDevices( + final PipeDeleteDevicesPlan pipeDeleteDevicesPlan, final C context) { + return visitPlan(pipeDeleteDevicesPlan, context); + } + + public R visitSetTableComment(final SetTableCommentPlan setTableCommentPlan, final C context) { + return visitPlan(setTableCommentPlan, context); + } + + // Use set table by default + public R visitSetViewComment(final SetViewCommentPlan setViewCommentPlan, final C context) { + return visitSetTableComment(setViewCommentPlan, context); + } + + public R visitSetTableColumnComment( + final SetTableColumnCommentPlan setTableColumnCommentPlan, final C context) { + return visitPlan(setTableColumnCommentPlan, context); + } + + public R visitRenameTable(final RenameTablePlan renameTablePlan, final C context) { + return visitPlan(renameTablePlan, context); + } + + // Use rename table by default + public R visitRenameView(final RenameViewPlan renameViewPlan, final C context) { + return visitRenameTable(renameViewPlan, context); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/auth/AuthorPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/auth/AuthorPlan.java deleted file mode 100644 index be162647f8fca..0000000000000 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/auth/AuthorPlan.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.consensus.request.auth; - -import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; - -import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.slf4j.Logger; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; - -public class AuthorPlan extends ConfigPhysicalPlan { - private static final Logger logger = org.slf4j.LoggerFactory.getLogger(AuthorPlan.class); - - private ConfigPhysicalPlanType authorType; - private String roleName; - private String password; - private String newPassword; - private Set permissions; - private List nodeNameList; - private String userName; - private boolean grantOpt; - - public AuthorPlan(final ConfigPhysicalPlanType type) { - super(type); - authorType = type; - } - - /** - * {@link AuthorPlan} Constructor. - * - * @param authorType author type - * @param userName user name - * @param roleName role name - * @param password password - * @param newPassword new password - * @param permissions permissions - * @param grantOpt with grant option, only grant statement can set grantOpt = true - * @param nodeNameList node name in Path structure - */ - public AuthorPlan( - final ConfigPhysicalPlanType authorType, - final String userName, - final String roleName, - final String password, - final String newPassword, - final Set permissions, - final boolean grantOpt, - final List nodeNameList) { - this(authorType); - this.authorType = authorType; - this.userName = userName; - this.roleName = roleName; - this.password = password; - this.newPassword = newPassword; - this.permissions = permissions; - this.grantOpt = grantOpt; - this.nodeNameList = nodeNameList; - } - - public ConfigPhysicalPlanType getAuthorType() { - return authorType; - } - - public void setAuthorType(ConfigPhysicalPlanType authorType) { - this.authorType = authorType; - } - - public String getRoleName() { - return roleName; - } - - public void setRoleName(String roleName) { - this.roleName = roleName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getNewPassword() { - return newPassword; - } - - public void setNewPassword(String newPassword) { - this.newPassword = newPassword; - } - - public Set getPermissions() { - return permissions; - } - - public void setPermissions(Set permissions) { - this.permissions = permissions; - } - - public boolean getGrantOpt() { - return this.grantOpt; - } - - public void setGrantOpt(boolean grantOpt) { - this.grantOpt = grantOpt; - } - - public List getNodeNameList() { - return nodeNameList; - } - - public void setNodeNameList(List nodeNameList) { - this.nodeNameList = nodeNameList; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(getPlanType(authorType), stream); - BasicStructureSerDeUtil.write(userName, stream); - BasicStructureSerDeUtil.write(roleName, stream); - BasicStructureSerDeUtil.write(password, stream); - BasicStructureSerDeUtil.write(newPassword, stream); - if (permissions == null) { - stream.write((byte) 0); - } else { - stream.write((byte) 1); - stream.writeInt(permissions.size()); - for (int permission : permissions) { - stream.writeInt(permission); - } - } - BasicStructureSerDeUtil.write(nodeNameList.size(), stream); - for (PartialPath partialPath : nodeNameList) { - BasicStructureSerDeUtil.write(partialPath.getFullPath(), stream); - } - BasicStructureSerDeUtil.write(grantOpt ? 1 : 0, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) { - userName = BasicStructureSerDeUtil.readString(buffer); - roleName = BasicStructureSerDeUtil.readString(buffer); - password = BasicStructureSerDeUtil.readString(buffer); - newPassword = BasicStructureSerDeUtil.readString(buffer); - final byte hasPermissions = buffer.get(); - if (hasPermissions == (byte) 0) { - this.permissions = null; - } else { - int permissionsSize = buffer.getInt(); - this.permissions = new HashSet<>(); - for (int i = 0; i < permissionsSize; i++) { - permissions.add(buffer.getInt()); - } - } - - int nodeNameListSize = BasicStructureSerDeUtil.readInt(buffer); - nodeNameList = new ArrayList<>(nodeNameListSize); - try { - for (int i = 0; i < nodeNameListSize; i++) { - nodeNameList.add(new PartialPath(BasicStructureSerDeUtil.readString(buffer))); - } - } catch (MetadataException e) { - logger.error("Invalid path when deserialize authPlan: {}", nodeNameList, e); - } - grantOpt = false; - if (this.authorType.ordinal() >= ConfigPhysicalPlanType.CreateUser.ordinal()) { - grantOpt = BasicStructureSerDeUtil.readInt(buffer) > 0; - } - } - - private short getPlanType(ConfigPhysicalPlanType configPhysicalPlanType) { - short type; - switch (configPhysicalPlanType) { - case CreateUser: - type = ConfigPhysicalPlanType.CreateUser.getPlanType(); - break; - case CreateRole: - type = ConfigPhysicalPlanType.CreateRole.getPlanType(); - break; - case DropUser: - type = ConfigPhysicalPlanType.DropUser.getPlanType(); - break; - case DropRole: - type = ConfigPhysicalPlanType.DropRole.getPlanType(); - break; - case GrantRole: - type = ConfigPhysicalPlanType.GrantRole.getPlanType(); - break; - case GrantUser: - type = ConfigPhysicalPlanType.GrantUser.getPlanType(); - break; - case GrantRoleToUser: - type = ConfigPhysicalPlanType.GrantRoleToUser.getPlanType(); - break; - case RevokeUser: - type = ConfigPhysicalPlanType.RevokeUser.getPlanType(); - break; - case RevokeRole: - type = ConfigPhysicalPlanType.RevokeRole.getPlanType(); - break; - case RevokeRoleFromUser: - type = ConfigPhysicalPlanType.RevokeRoleFromUser.getPlanType(); - break; - case UpdateUser: - type = ConfigPhysicalPlanType.UpdateUser.getPlanType(); - break; - case ListUser: - type = ConfigPhysicalPlanType.ListUser.getPlanType(); - break; - case ListRole: - type = ConfigPhysicalPlanType.ListRole.getPlanType(); - break; - case ListUserPrivilege: - type = ConfigPhysicalPlanType.ListUserPrivilege.getPlanType(); - break; - case ListRolePrivilege: - type = ConfigPhysicalPlanType.ListRolePrivilege.getPlanType(); - break; - case ListUserRoles: - type = ConfigPhysicalPlanType.ListUserRoles.getPlanType(); - break; - case ListRoleUsers: - type = ConfigPhysicalPlanType.ListRoleUsers.getPlanType(); - break; - case CreateUserWithRawPassword: - type = ConfigPhysicalPlanType.CreateUserWithRawPassword.getPlanType(); - break; - default: - throw new IllegalArgumentException("Unknown operator: " + configPhysicalPlanType); - } - return type; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - AuthorPlan that = (AuthorPlan) o; - return Objects.equals(authorType, that.authorType) - && Objects.equals(userName, that.userName) - && Objects.equals(roleName, that.roleName) - && Objects.equals(password, that.password) - && Objects.equals(newPassword, that.newPassword) - && Objects.equals(permissions, that.permissions) - && grantOpt == that.grantOpt - && Objects.equals(nodeNameList, that.nodeNameList); - } - - @Override - public int hashCode() { - return Objects.hash( - authorType, userName, roleName, password, newPassword, permissions, nodeNameList, grantOpt); - } - - @Override - public String toString() { - return "[type:" - + authorType - + ", username:" - + userName - + ", rolename:" - + roleName - + ", permissions:" - + PrivilegeType.toPriType(permissions) - + ", grant option:" - + grantOpt - + ", paths:" - + nodeNameList - + "]"; - } -} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java new file mode 100644 index 0000000000000..032c21a731c0d --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public abstract class ConfigPhysicalReadPlan extends ConfigPhysicalPlan { + + protected ConfigPhysicalReadPlan(final ConfigPhysicalPlanType type) { + super(type); + } + + @Override + protected void serializeImpl(final DataOutputStream stream) throws IOException { + // Read request does not need to be serialized + } + + @Override + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + // Read request does not need to be deserialized + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java new file mode 100644 index 0000000000000..303f30b9c7792 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.ainode; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class GetAINodeConfigurationPlan extends ConfigPhysicalReadPlan { + + // if aiNodeId is set to -1, return all AINode configurations. + private int aiNodeId; + + public GetAINodeConfigurationPlan() { + super(ConfigPhysicalPlanType.GetAINodeConfiguration); + } + + public GetAINodeConfigurationPlan(final int aiNodeId) { + super(ConfigPhysicalPlanType.GetAINodeConfiguration); + this.aiNodeId = aiNodeId; + } + + public int getAiNodeId() { + return aiNodeId; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof GetAINodeConfigurationPlan)) { + return false; + } + final GetAINodeConfigurationPlan that = (GetAINodeConfigurationPlan) o; + return aiNodeId == that.aiNodeId; + } + + @Override + public int hashCode() { + return Integer.hashCode(aiNodeId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java new file mode 100644 index 0000000000000..5217849deb488 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.cq; + +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +import static org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.SHOW_CQ; + +public class ShowCQPlan extends ConfigPhysicalReadPlan { + + public ShowCQPlan() { + super(SHOW_CQ); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/CountDatabasePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/CountDatabasePlan.java index 3e838c08a0698..b32fef1a36f16 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/CountDatabasePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/CountDatabasePlan.java @@ -20,40 +20,37 @@ package org.apache.iotdb.confignode.consensus.request.read.database; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; -public class CountDatabasePlan extends ConfigPhysicalPlan { +public class CountDatabasePlan extends ConfigPhysicalReadPlan { - private String[] storageGroupPattern; - private PathPatternTree scope; + private final String[] storageGroupPattern; + private final PathPatternTree scope; + private final boolean isTableModel; - public CountDatabasePlan() { + public CountDatabasePlan( + final List storageGroupPattern, + final PathPatternTree scope, + final boolean isTableModel) { super(ConfigPhysicalPlanType.CountDatabase); - } - - public CountDatabasePlan(ConfigPhysicalPlanType type) { - super(type); - } - - public CountDatabasePlan(List storageGroupPattern, PathPatternTree scope) { - this(); this.storageGroupPattern = storageGroupPattern.toArray(new String[0]); this.scope = scope; + this.isTableModel = isTableModel; } public CountDatabasePlan( - ConfigPhysicalPlanType type, List storageGroupPattern, PathPatternTree scope) { + final ConfigPhysicalPlanType type, + final List storageGroupPattern, + final PathPatternTree scope, + final boolean isTableModel) { super(type); this.storageGroupPattern = storageGroupPattern.toArray(new String[0]); this.scope = scope; + this.isTableModel = isTableModel; } public String[] getDatabasePattern() { @@ -64,36 +61,19 @@ public PathPatternTree getScope() { return scope; } - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - stream.writeInt(storageGroupPattern.length); - for (String node : storageGroupPattern) { - BasicStructureSerDeUtil.write(node, stream); - } - scope.serialize(stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) { - int length = buffer.getInt(); - storageGroupPattern = new String[length]; - for (int i = 0; i < length; i++) { - storageGroupPattern[i] = BasicStructureSerDeUtil.readString(buffer); - } - scope = PathPatternTree.deserialize(buffer); + public boolean isTableModel() { + return isTableModel; } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - CountDatabasePlan that = (CountDatabasePlan) o; + final CountDatabasePlan that = (CountDatabasePlan) o; return Arrays.equals(storageGroupPattern, that.storageGroupPattern); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/GetDatabasePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/GetDatabasePlan.java index 161e58776f5b3..b0adfe88d8981 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/GetDatabasePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/database/GetDatabasePlan.java @@ -25,12 +25,18 @@ import java.util.List; public class GetDatabasePlan extends CountDatabasePlan { + private final boolean isShowDatabasePlan; - public GetDatabasePlan() { - super(ConfigPhysicalPlanType.GetDatabase); + public GetDatabasePlan( + final List storageGroupPathPattern, + final PathPatternTree scope, + final boolean isTableModel, + final boolean isShowDatabasePlan) { + super(ConfigPhysicalPlanType.GetDatabase, storageGroupPathPattern, scope, isTableModel); + this.isShowDatabasePlan = isShowDatabasePlan; } - public GetDatabasePlan(List storageGroupPathPattern, PathPatternTree scope) { - super(ConfigPhysicalPlanType.GetDatabase, storageGroupPathPattern, scope); + public boolean isShowDatabasePlan() { + return isShowDatabasePlan; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/datanode/GetDataNodeConfigurationPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/datanode/GetDataNodeConfigurationPlan.java index 22f7202841d50..7223415c3de09 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/datanode/GetDataNodeConfigurationPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/datanode/GetDataNodeConfigurationPlan.java @@ -19,25 +19,18 @@ package org.apache.iotdb.confignode.consensus.request.read.datanode; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; /** Get DataNodeInfo by the specific DataNode's id. And return all when dataNodeID is set to -1. */ -public class GetDataNodeConfigurationPlan extends ConfigPhysicalPlan { +public class GetDataNodeConfigurationPlan extends ConfigPhysicalReadPlan { - private int dataNodeId; + private final int dataNodeId; - public GetDataNodeConfigurationPlan() { + public GetDataNodeConfigurationPlan(final int dataNodeId) { super(ConfigPhysicalPlanType.GetDataNodeConfiguration); - } - - public GetDataNodeConfigurationPlan(int dataNodeId) { - this(); this.dataNodeId = dataNodeId; } @@ -46,25 +39,14 @@ public Integer getDataNodeId() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - stream.writeInt(dataNodeId); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) { - this.dataNodeId = buffer.getInt(); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetDataNodeConfigurationPlan that = (GetDataNodeConfigurationPlan) o; + final GetDataNodeConfigurationPlan that = (GetDataNodeConfigurationPlan) o; return dataNodeId == that.dataNodeId; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetAllFunctionTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetAllFunctionTablePlan.java new file mode 100644 index 0000000000000..a2f8d65ef1be0 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetAllFunctionTablePlan.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.function; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class GetAllFunctionTablePlan extends ConfigPhysicalReadPlan { + + public GetAllFunctionTablePlan() { + super(ConfigPhysicalPlanType.GetAllFunctionTable); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetFunctionTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetFunctionTablePlan.java index 0dc530bb7d46b..3e36f6568859e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetFunctionTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetFunctionTablePlan.java @@ -19,26 +19,20 @@ package org.apache.iotdb.confignode.consensus.request.read.function; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; +public class GetFunctionTablePlan extends ConfigPhysicalReadPlan { -public class GetFunctionTablePlan extends ConfigPhysicalPlan { + private final Model model; - public GetFunctionTablePlan() { + public GetFunctionTablePlan(Model model) { super(ConfigPhysicalPlanType.GetFunctionTable); + this.model = model; } - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // do nothing + public Model getModel() { + return model; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetUDFJarPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetUDFJarPlan.java index 09baafdcc4f41..ba81d964360a2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetUDFJarPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/function/GetUDFJarPlan.java @@ -19,27 +19,17 @@ package org.apache.iotdb.confignode.consensus.request.read.function; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import java.util.Objects; -public class GetUDFJarPlan extends ConfigPhysicalPlan { - - private List jarNames; +public class GetUDFJarPlan extends ConfigPhysicalReadPlan { - public GetUDFJarPlan() { - super(ConfigPhysicalPlanType.GetFunctionJar); - } + private final List jarNames; - public GetUDFJarPlan(List triggerNames) { + public GetUDFJarPlan(final List triggerNames) { super(ConfigPhysicalPlanType.GetFunctionJar); jarNames = triggerNames; } @@ -49,27 +39,7 @@ public List getJarNames() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(jarNames.size(), stream); - for (String jarName : jarNames) { - ReadWriteIOUtils.write(jarName, stream); - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int size = ReadWriteIOUtils.readInt(buffer); - jarNames = new ArrayList<>(size); - while (size > 0) { - jarNames.add(ReadWriteIOUtils.readString(buffer)); - size--; - } - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } @@ -79,7 +49,7 @@ public boolean equals(Object o) { if (!super.equals(o)) { return false; } - GetUDFJarPlan that = (GetUDFJarPlan) o; + final GetUDFJarPlan that = (GetUDFJarPlan) o; return Objects.equals(jarNames, that.jarNames); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java new file mode 100644 index 0000000000000..dd79910e51fa8 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.model; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; +import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq; + +import java.util.Objects; + +public class GetModelInfoPlan extends ConfigPhysicalReadPlan { + + private String modelId; + + public GetModelInfoPlan() { + super(ConfigPhysicalPlanType.GetModelInfo); + } + + public GetModelInfoPlan(final TGetModelInfoReq getModelInfoReq) { + super(ConfigPhysicalPlanType.GetModelInfo); + this.modelId = getModelInfoReq.getModelId(); + } + + public String getModelId() { + return modelId; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final GetModelInfoPlan that = (GetModelInfoPlan) o; + return Objects.equals(modelId, that.modelId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), modelId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java new file mode 100644 index 0000000000000..16bc16bc8725d --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.model; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; +import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq; + +import java.util.Objects; + +public class ShowModelPlan extends ConfigPhysicalReadPlan { + + private String modelName; + + public ShowModelPlan() { + super(ConfigPhysicalPlanType.ShowModel); + } + + public ShowModelPlan(final TShowModelReq showModelReq) { + super(ConfigPhysicalPlanType.ShowModel); + if (showModelReq.isSetModelId()) { + this.modelName = showModelReq.getModelId(); + } + } + + public boolean isSetModelName() { + return modelName != null; + } + + public String getModelName() { + return modelName; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final ShowModelPlan that = (ShowModelPlan) o; + return Objects.equals(modelName, that.modelName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), modelName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/CountTimeSlotListPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/CountTimeSlotListPlan.java index acf7c974224ed..0050f712fb038 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/CountTimeSlotListPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/CountTimeSlotListPlan.java @@ -22,18 +22,12 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class CountTimeSlotListPlan extends ConfigPhysicalPlan { +public class CountTimeSlotListPlan extends ConfigPhysicalReadPlan { private String database; @@ -41,16 +35,12 @@ public class CountTimeSlotListPlan extends ConfigPhysicalPlan { private TConsensusGroupId regionId; - private long startTime; + private final long startTime; - private long endTime; + private final long endTime; - public CountTimeSlotListPlan() { + public CountTimeSlotListPlan(final long startTime, final long endTime) { super(ConfigPhysicalPlanType.CountTimeSlotList); - } - - public CountTimeSlotListPlan(long startTime, long endTime) { - this(); this.startTime = startTime; this.endTime = endTime; this.database = ""; @@ -58,7 +48,7 @@ public CountTimeSlotListPlan(long startTime, long endTime) { this.regionId = new TConsensusGroupId(TConsensusGroupType.DataRegion, -1); } - public void setDatabase(String database) { + public void setDatabase(final String database) { this.database = database; } @@ -66,7 +56,7 @@ public String getDatabase() { return database; } - public void setRegionId(TConsensusGroupId regionId) { + public void setRegionId(final TConsensusGroupId regionId) { this.regionId = regionId; } @@ -74,7 +64,7 @@ public TConsensusGroupId getRegionId() { return regionId; } - public void setSeriesSlotId(TSeriesPartitionSlot seriesSlotId) { + public void setSeriesSlotId(final TSeriesPartitionSlot seriesSlotId) { this.seriesSlotId = seriesSlotId; } @@ -91,33 +81,14 @@ public long getEndTime() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(database, stream); - ThriftCommonsSerDeUtils.serializeTSeriesPartitionSlot(seriesSlotId, stream); - ThriftCommonsSerDeUtils.serializeTConsensusGroupId(regionId, stream); - stream.writeLong(startTime); - stream.writeLong(endTime); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.seriesSlotId = ThriftCommonsSerDeUtils.deserializeTSeriesPartitionSlot(buffer); - this.regionId = ThriftCommonsSerDeUtils.deserializeTConsensusGroupId(buffer); - this.startTime = buffer.getLong(); - this.endTime = buffer.getLong(); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - CountTimeSlotListPlan that = (CountTimeSlotListPlan) o; + final CountTimeSlotListPlan that = (CountTimeSlotListPlan) o; return database.equals(that.database) && seriesSlotId.equals(that.seriesSlotId) && regionId.equals(that.regionId) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetDataPartitionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetDataPartitionPlan.java index 32e494b55344d..cce8e14611490 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetDataPartitionPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetDataPartitionPlan.java @@ -20,39 +20,28 @@ package org.apache.iotdb.confignode.consensus.request.read.partition; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; -import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; /** Get or create DataPartition by the specific partitionSlotsMap. */ -public class GetDataPartitionPlan extends ConfigPhysicalPlan { +public class GetDataPartitionPlan extends ConfigPhysicalReadPlan { // Map>> protected Map> partitionSlotsMap; - public GetDataPartitionPlan() { - super(ConfigPhysicalPlanType.GetDataPartition); - } - - public GetDataPartitionPlan(ConfigPhysicalPlanType configPhysicalPlanType) { + public GetDataPartitionPlan(final ConfigPhysicalPlanType configPhysicalPlanType) { super(configPhysicalPlanType); } public GetDataPartitionPlan( - Map> partitionSlotsMap) { - this(); + final Map> partitionSlotsMap) { + super(ConfigPhysicalPlanType.GetDataPartition); this.partitionSlotsMap = partitionSlotsMap; } @@ -66,58 +55,19 @@ public Map> getPartitionSlotsMa * @param req TDataPartitionReq * @return GetDataPartitionPlan */ - public static GetDataPartitionPlan convertFromRpcTDataPartitionReq(TDataPartitionReq req) { + public static GetDataPartitionPlan convertFromRpcTDataPartitionReq(final TDataPartitionReq req) { return new GetDataPartitionPlan(new ConcurrentHashMap<>(req.getPartitionSlotsMap())); } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - stream.writeInt(partitionSlotsMap.size()); - for (Entry> entry : - partitionSlotsMap.entrySet()) { - String storageGroup = entry.getKey(); - Map seriesPartitionTimePartitionSlots = entry.getValue(); - BasicStructureSerDeUtil.write(storageGroup, stream); - stream.writeInt(seriesPartitionTimePartitionSlots.size()); - for (Entry e : - seriesPartitionTimePartitionSlots.entrySet()) { - TSeriesPartitionSlot seriesPartitionSlot = e.getKey(); - TTimeSlotList timePartitionSlotList = e.getValue(); - ThriftCommonsSerDeUtils.serializeTSeriesPartitionSlot(seriesPartitionSlot, stream); - ThriftCommonsSerDeUtils.serializeTTimePartitionSlotList(timePartitionSlotList, stream); - } - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) { - partitionSlotsMap = new HashMap<>(); - int storageGroupNum = buffer.getInt(); - for (int i = 0; i < storageGroupNum; i++) { - String storageGroup = BasicStructureSerDeUtil.readString(buffer); - partitionSlotsMap.put(storageGroup, new HashMap<>()); - int seriesPartitionSlotNum = buffer.getInt(); - for (int j = 0; j < seriesPartitionSlotNum; j++) { - TSeriesPartitionSlot seriesPartitionSlot = - ThriftCommonsSerDeUtils.deserializeTSeriesPartitionSlot(buffer); - TTimeSlotList timePartitionSlotList = - ThriftCommonsSerDeUtils.deserializeTTimePartitionSlotList(buffer); - partitionSlotsMap.get(storageGroup).put(seriesPartitionSlot, timePartitionSlotList); - } - } - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetDataPartitionPlan that = (GetDataPartitionPlan) o; + final GetDataPartitionPlan that = (GetDataPartitionPlan) o; return partitionSlotsMap.equals(that.partitionSlotsMap); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetNodePathsPartitionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetNodePathsPartitionPlan.java index 978a70853b26e..3b8d0a7c8b19a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetNodePathsPartitionPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetNodePathsPartitionPlan.java @@ -20,17 +20,13 @@ package org.apache.iotdb.confignode.consensus.request.read.partition; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetNodePathsPartitionPlan extends ConfigPhysicalPlan { +public class GetNodePathsPartitionPlan extends ConfigPhysicalReadPlan { private PartialPath partialPath; private PathPatternTree scope; private int level = -1; @@ -43,7 +39,7 @@ public PathPatternTree getScope() { return scope; } - public void setScope(PathPatternTree scope) { + public void setScope(final PathPatternTree scope) { this.scope = scope; } @@ -51,7 +47,7 @@ public PartialPath getPartialPath() { return partialPath; } - public void setPartialPath(PartialPath partialPath) { + public void setPartialPath(final PartialPath partialPath) { this.partialPath = partialPath; } @@ -59,34 +55,19 @@ public int getLevel() { return level; } - public void setLevel(int level) { + public void setLevel(final int level) { this.level = level; } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - partialPath.serialize(stream); - scope.serialize(stream); - stream.writeInt(level); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - partialPath = (PartialPath) PathDeserializeUtil.deserialize(buffer); - scope = PathPatternTree.deserialize(buffer); - level = buffer.getInt(); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetNodePathsPartitionPlan that = (GetNodePathsPartitionPlan) o; + final GetNodePathsPartitionPlan that = (GetNodePathsPartitionPlan) o; return level == that.level && Objects.equals(partialPath, that.partialPath); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateDataPartitionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateDataPartitionPlan.java index 347ae31214700..800b7e0e19a49 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateDataPartitionPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateDataPartitionPlan.java @@ -29,13 +29,9 @@ public class GetOrCreateDataPartitionPlan extends GetDataPartitionPlan { - public GetOrCreateDataPartitionPlan() { - super(ConfigPhysicalPlanType.GetOrCreateDataPartition); - } - public GetOrCreateDataPartitionPlan( - Map> partitionSlotsMap) { - this(); + final Map> partitionSlotsMap) { + super(ConfigPhysicalPlanType.GetOrCreateDataPartition); this.partitionSlotsMap = partitionSlotsMap; } @@ -46,7 +42,7 @@ public GetOrCreateDataPartitionPlan( * @return GetOrCreateDataPartitionPlan */ public static GetOrCreateDataPartitionPlan convertFromRpcTDataPartitionReq( - TDataPartitionReq req) { + final TDataPartitionReq req) { return new GetOrCreateDataPartitionPlan(new ConcurrentHashMap<>(req.getPartitionSlotsMap())); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateSchemaPartitionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateSchemaPartitionPlan.java index e245bc7155f7e..c7b7272c569ec 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateSchemaPartitionPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetOrCreateSchemaPartitionPlan.java @@ -31,7 +31,8 @@ public GetOrCreateSchemaPartitionPlan() { super(ConfigPhysicalPlanType.GetOrCreateSchemaPartition); } - public GetOrCreateSchemaPartitionPlan(Map> partitionSlotsMap) { + public GetOrCreateSchemaPartitionPlan( + final Map> partitionSlotsMap) { this(); this.partitionSlotsMap = partitionSlotsMap; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSchemaPartitionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSchemaPartitionPlan.java index 2751c5efaed88..8bfccd18a1c62 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSchemaPartitionPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSchemaPartitionPlan.java @@ -20,39 +20,27 @@ package org.apache.iotdb.confignode.consensus.request.read.partition; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; -import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; /** Get or create SchemaPartition by the specific partitionSlotsMap. */ -public class GetSchemaPartitionPlan extends ConfigPhysicalPlan { +public class GetSchemaPartitionPlan extends ConfigPhysicalReadPlan { // Map> // Get all SchemaPartitions when the partitionSlotsMap is empty // Get all exists SchemaPartitions in one StorageGroup when the SeriesPartitionSlot is empty protected Map> partitionSlotsMap; - public GetSchemaPartitionPlan() { - super(ConfigPhysicalPlanType.GetSchemaPartition); - } - - public GetSchemaPartitionPlan(ConfigPhysicalPlanType configPhysicalPlanType) { + public GetSchemaPartitionPlan(final ConfigPhysicalPlanType configPhysicalPlanType) { super(configPhysicalPlanType); } - public GetSchemaPartitionPlan(Map> partitionSlotsMap) { - this(); + public GetSchemaPartitionPlan(final Map> partitionSlotsMap) { + super(ConfigPhysicalPlanType.GetSchemaPartition); this.partitionSlotsMap = partitionSlotsMap; } @@ -61,46 +49,14 @@ public Map> getPartitionSlotsMap() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - stream.writeInt(partitionSlotsMap.size()); - for (Entry> entry : partitionSlotsMap.entrySet()) { - String storageGroup = entry.getKey(); - List seriesPartitionSlots = entry.getValue(); - BasicStructureSerDeUtil.write(storageGroup, stream); - stream.writeInt(seriesPartitionSlots.size()); - seriesPartitionSlots.forEach( - seriesPartitionSlot -> - ThriftCommonsSerDeUtils.serializeTSeriesPartitionSlot(seriesPartitionSlot, stream)); - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - partitionSlotsMap = new HashMap<>(); - int storageGroupNum = buffer.getInt(); - for (int i = 0; i < storageGroupNum; i++) { - String storageGroup = BasicStructureSerDeUtil.readString(buffer); - partitionSlotsMap.put(storageGroup, new ArrayList<>()); - int seriesPartitionSlotNum = buffer.getInt(); - for (int j = 0; j < seriesPartitionSlotNum; j++) { - TSeriesPartitionSlot seriesPartitionSlot = - ThriftCommonsSerDeUtils.deserializeTSeriesPartitionSlot(buffer); - partitionSlotsMap.get(storageGroup).add(seriesPartitionSlot); - } - } - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetSchemaPartitionPlan that = (GetSchemaPartitionPlan) o; + final GetSchemaPartitionPlan that = (GetSchemaPartitionPlan) o; return partitionSlotsMap.equals(that.partitionSlotsMap); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSeriesSlotListPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSeriesSlotListPlan.java index 832a9b8fa39ba..f0d130fa5636d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSeriesSlotListPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetSeriesSlotListPlan.java @@ -20,28 +20,19 @@ package org.apache.iotdb.confignode.consensus.request.read.partition; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetSeriesSlotListPlan extends ConfigPhysicalPlan { +public class GetSeriesSlotListPlan extends ConfigPhysicalReadPlan { - private String database; + private final String database; - private TConsensusGroupType partitionType; + private final TConsensusGroupType partitionType; - public GetSeriesSlotListPlan() { + public GetSeriesSlotListPlan(final String database, final TConsensusGroupType partitionType) { super(ConfigPhysicalPlanType.GetSeriesSlotList); - } - - public GetSeriesSlotListPlan(String database, TConsensusGroupType partitionType) { - this(); this.database = database; this.partitionType = partitionType; } @@ -55,27 +46,14 @@ public TConsensusGroupType getPartitionType() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(database, stream); - stream.writeInt(partitionType.ordinal()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.partitionType = TConsensusGroupType.findByValue(buffer.getInt()); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetSeriesSlotListPlan that = (GetSeriesSlotListPlan) o; + final GetSeriesSlotListPlan that = (GetSeriesSlotListPlan) o; return database.equals(that.database) && partitionType.equals(that.partitionType); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetTimeSlotListPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetTimeSlotListPlan.java index 1b6f881763e95..00a130fc307e2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetTimeSlotListPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/partition/GetTimeSlotListPlan.java @@ -22,18 +22,12 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetTimeSlotListPlan extends ConfigPhysicalPlan { +public class GetTimeSlotListPlan extends ConfigPhysicalReadPlan { private String database; @@ -41,16 +35,12 @@ public class GetTimeSlotListPlan extends ConfigPhysicalPlan { private TConsensusGroupId regionId; - private long startTime; + private final long startTime; - private long endTime; + private final long endTime; - public GetTimeSlotListPlan() { + public GetTimeSlotListPlan(final long startTime, final long endTime) { super(ConfigPhysicalPlanType.GetTimeSlotList); - } - - public GetTimeSlotListPlan(long startTime, long endTime) { - this(); this.startTime = startTime; this.endTime = endTime; this.database = ""; @@ -58,7 +48,7 @@ public GetTimeSlotListPlan(long startTime, long endTime) { this.regionId = new TConsensusGroupId(TConsensusGroupType.DataRegion, -1); } - public void setDatabase(String database) { + public void setDatabase(final String database) { this.database = database; } @@ -66,7 +56,7 @@ public String getDatabase() { return database; } - public void setRegionId(TConsensusGroupId regionId) { + public void setRegionId(final TConsensusGroupId regionId) { this.regionId = regionId; } @@ -74,7 +64,7 @@ public TConsensusGroupId getRegionId() { return regionId; } - public void setSeriesSlotId(TSeriesPartitionSlot seriesSlotId) { + public void setSeriesSlotId(final TSeriesPartitionSlot seriesSlotId) { this.seriesSlotId = seriesSlotId; } @@ -91,33 +81,14 @@ public long getEndTime() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(database, stream); - ThriftCommonsSerDeUtils.serializeTSeriesPartitionSlot(seriesSlotId, stream); - ThriftCommonsSerDeUtils.serializeTConsensusGroupId(regionId, stream); - stream.writeLong(startTime); - stream.writeLong(endTime); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.seriesSlotId = ThriftCommonsSerDeUtils.deserializeTSeriesPartitionSlot(buffer); - this.regionId = ThriftCommonsSerDeUtils.deserializeTConsensusGroupId(buffer); - this.startTime = buffer.getLong(); - this.endTime = buffer.getLong(); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetTimeSlotListPlan that = (GetTimeSlotListPlan) o; + final GetTimeSlotListPlan that = (GetTimeSlotListPlan) o; return database.equals(that.database) && seriesSlotId.equals(that.seriesSlotId) && regionId.equals(that.regionId) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginJarPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginJarPlan.java index 4b5b2152db925..a8f99e6a3d105 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginJarPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginJarPlan.java @@ -19,25 +19,15 @@ package org.apache.iotdb.confignode.consensus.request.read.pipe.plugin; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; -public class GetPipePluginJarPlan extends ConfigPhysicalPlan { - private List jarNames; +public class GetPipePluginJarPlan extends ConfigPhysicalReadPlan { + private final List jarNames; - public GetPipePluginJarPlan() { - super(ConfigPhysicalPlanType.GetPipePluginJar); - } - - public GetPipePluginJarPlan(List jarNames) { + public GetPipePluginJarPlan(final List jarNames) { super(ConfigPhysicalPlanType.GetPipePluginJar); this.jarNames = jarNames; } @@ -45,23 +35,4 @@ public GetPipePluginJarPlan(List jarNames) { public List getJarNames() { return jarNames; } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(jarNames.size(), stream); - for (String jarName : jarNames) { - ReadWriteIOUtils.write(jarName, stream); - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int size = ReadWriteIOUtils.readInt(buffer); - jarNames = new ArrayList<>(); - for (int i = 0; i < size; i++) { - jarNames.add(ReadWriteIOUtils.readString(buffer)); - } - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginTablePlan.java index cabe5cc1139b9..39cf23ed34436 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/plugin/GetPipePluginTablePlan.java @@ -19,26 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.pipe.plugin; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class GetPipePluginTablePlan extends ConfigPhysicalPlan { +public class GetPipePluginTablePlan extends ConfigPhysicalReadPlan { public GetPipePluginTablePlan() { super(ConfigPhysicalPlanType.GetPipePluginTable); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // Empty method, since it is not needed now - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/task/ShowPipePlanV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/task/ShowPipePlanV2.java index 1d1acf293f3e0..6bb67d2414620 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/task/ShowPipePlanV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/task/ShowPipePlanV2.java @@ -19,26 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.pipe.task; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class ShowPipePlanV2 extends ConfigPhysicalPlan { +public class ShowPipePlanV2 extends ConfigPhysicalReadPlan { public ShowPipePlanV2() { super(ConfigPhysicalPlanType.ShowPipeV2); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // Empty method, since it is not needed now - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionIdPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionIdPlan.java index 00797a1fb149a..878d18c3bab21 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionIdPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionIdPlan.java @@ -22,22 +22,16 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetRegionIdPlan extends ConfigPhysicalPlan { +public class GetRegionIdPlan extends ConfigPhysicalReadPlan { private String database; - private TConsensusGroupType partitionType; + private final TConsensusGroupType partitionType; private TTimePartitionSlot startTimeSlotId; @@ -45,12 +39,8 @@ public class GetRegionIdPlan extends ConfigPhysicalPlan { private TSeriesPartitionSlot seriesSlotId; - public GetRegionIdPlan() { + public GetRegionIdPlan(final TConsensusGroupType partitionType) { super(ConfigPhysicalPlanType.GetRegionId); - } - - public GetRegionIdPlan(TConsensusGroupType partitionType) { - this(); this.partitionType = partitionType; this.database = ""; this.seriesSlotId = new TSeriesPartitionSlot(-1); @@ -62,11 +52,11 @@ public String getDatabase() { return database; } - public void setDatabase(String database) { + public void setDatabase(final String database) { this.database = database; } - public void setSeriesSlotId(TSeriesPartitionSlot seriesSlotId) { + public void setSeriesSlotId(final TSeriesPartitionSlot seriesSlotId) { this.seriesSlotId = seriesSlotId; } @@ -78,7 +68,7 @@ public TTimePartitionSlot getStartTimeSlotId() { return startTimeSlotId; } - public void setStartTimeSlotId(TTimePartitionSlot startTimeSlotId) { + public void setStartTimeSlotId(final TTimePartitionSlot startTimeSlotId) { this.startTimeSlotId = startTimeSlotId; } @@ -86,7 +76,7 @@ public TTimePartitionSlot getEndTimeSlotId() { return endTimeSlotId; } - public void setEndTimeSlotId(TTimePartitionSlot endTimeSlotId) { + public void setEndTimeSlotId(final TTimePartitionSlot endTimeSlotId) { this.endTimeSlotId = endTimeSlotId; } @@ -95,33 +85,14 @@ public TConsensusGroupType getPartitionType() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - stream.writeInt(partitionType.ordinal()); - ReadWriteIOUtils.write(database, stream); - ThriftCommonsSerDeUtils.serializeTSeriesPartitionSlot(seriesSlotId, stream); - ThriftCommonsSerDeUtils.serializeTTimePartitionSlot(startTimeSlotId, stream); - ThriftCommonsSerDeUtils.serializeTTimePartitionSlot(endTimeSlotId, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.partitionType = TConsensusGroupType.findByValue(buffer.getInt()); - this.database = ReadWriteIOUtils.readString(buffer); - this.seriesSlotId = ThriftCommonsSerDeUtils.deserializeTSeriesPartitionSlot(buffer); - this.startTimeSlotId = ThriftCommonsSerDeUtils.deserializeTTimePartitionSlot(buffer); - this.endTimeSlotId = ThriftCommonsSerDeUtils.deserializeTTimePartitionSlot(buffer); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetRegionIdPlan that = (GetRegionIdPlan) o; + final GetRegionIdPlan that = (GetRegionIdPlan) o; return database.equals(that.database) && partitionType.equals(that.partitionType) && seriesSlotId.equals(that.seriesSlotId) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionInfoListPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionInfoListPlan.java index f302fb03de35f..da504b36d6ce3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionInfoListPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/region/GetRegionInfoListPlan.java @@ -19,18 +19,11 @@ package org.apache.iotdb.confignode.consensus.request.read.region; -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class GetRegionInfoListPlan extends ConfigPhysicalPlan { +public class GetRegionInfoListPlan extends ConfigPhysicalReadPlan { private TShowRegionReq showRegionReq; @@ -38,7 +31,7 @@ public GetRegionInfoListPlan() { super(ConfigPhysicalPlanType.GetRegionInfoList); } - public GetRegionInfoListPlan(TShowRegionReq showRegionReq) { + public GetRegionInfoListPlan(final TShowRegionReq showRegionReq) { super(ConfigPhysicalPlanType.GetRegionInfoList); this.showRegionReq = showRegionReq; } @@ -47,39 +40,7 @@ public TShowRegionReq getShowRegionReq() { return showRegionReq; } - public void setShowRegionReq(TShowRegionReq showRegionReq) { + public void setShowRegionReq(final TShowRegionReq showRegionReq) { this.showRegionReq = showRegionReq; } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - stream.writeBoolean(showRegionReq != null); - if (showRegionReq != null) { - boolean setConsensusGroupType = showRegionReq.isSetConsensusGroupType(); - stream.writeBoolean(setConsensusGroupType); - if (setConsensusGroupType) { - ReadWriteIOUtils.write(showRegionReq.getConsensusGroupType().ordinal(), stream); - } - boolean setStorageGroups = showRegionReq.isSetDatabases(); - stream.writeBoolean(setStorageGroups); - if (setStorageGroups) { - ReadWriteIOUtils.writeStringList(showRegionReq.getDatabases(), stream); - } - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - if (ReadWriteIOUtils.readBool(buffer)) { - this.showRegionReq = new TShowRegionReq(); - if (ReadWriteIOUtils.readBool(buffer)) { - this.showRegionReq.setConsensusGroupType( - TConsensusGroupType.values()[ReadWriteIOUtils.readInt(buffer)]); - } - if (ReadWriteIOUtils.readBool(buffer)) { - this.showRegionReq.setDatabases(ReadWriteIOUtils.readStringList(buffer)); - } - } - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowSubscriptionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowSubscriptionPlan.java index 881a689539adb..5514e0542d9b0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowSubscriptionPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowSubscriptionPlan.java @@ -19,26 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.subscription; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class ShowSubscriptionPlan extends ConfigPhysicalPlan { +public class ShowSubscriptionPlan extends ConfigPhysicalReadPlan { public ShowSubscriptionPlan() { super(ConfigPhysicalPlanType.ShowSubscription); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // Empty method, since it is not needed now - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowTopicPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowTopicPlan.java index 810990ba9609d..7d1f5200aa110 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowTopicPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/subscription/ShowTopicPlan.java @@ -19,26 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.subscription; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class ShowTopicPlan extends ConfigPhysicalPlan { +public class ShowTopicPlan extends ConfigPhysicalReadPlan { public ShowTopicPlan() { super(ConfigPhysicalPlanType.ShowTopic); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // Empty method, since it is not needed now - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java new file mode 100644 index 0000000000000..27b7bcad87c29 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class DescTable4InformationSchemaPlan extends ConfigPhysicalReadPlan { + public DescTable4InformationSchemaPlan() { + super(ConfigPhysicalPlanType.DescTable4InformationSchema); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTablePlan.java new file mode 100644 index 0000000000000..ce0a6092c69e7 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTablePlan.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class DescTablePlan extends ConfigPhysicalReadPlan { + private final String database; + private final String tableName; + private final boolean isDetails; + + public DescTablePlan(final String database, final String tableName, final boolean isDetails) { + super(ConfigPhysicalPlanType.DescTable); + this.database = database; + this.tableName = tableName; + this.isDetails = isDetails; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public boolean isDetails() { + return isDetails; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/FetchTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/FetchTablePlan.java new file mode 100644 index 0000000000000..a69eda99d79af --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/FetchTablePlan.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +import java.util.Map; +import java.util.Set; + +public class FetchTablePlan extends ConfigPhysicalReadPlan { + + private final Map> fetchTableMap; + + public FetchTablePlan(final Map> fetchTableMap) { + super(ConfigPhysicalPlanType.FetchTable); + this.fetchTableMap = fetchTableMap; + } + + public Map> getFetchTableMap() { + return fetchTableMap; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java new file mode 100644 index 0000000000000..4dc452330436f --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class ShowTable4InformationSchemaPlan extends ConfigPhysicalReadPlan { + + public ShowTable4InformationSchemaPlan() { + super(ConfigPhysicalPlanType.ShowTable4InformationSchema); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java new file mode 100644 index 0000000000000..971a8757372b9 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class ShowTablePlan extends ConfigPhysicalReadPlan { + + private final String database; + + private final boolean isDetails; + + public ShowTablePlan(final String database, final boolean isDetails) { + super(ConfigPhysicalPlanType.ShowTable); + this.database = database; + this.isDetails = isDetails; + } + + public String getDatabase() { + return database; + } + + public boolean isDetails() { + return isDetails; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/CheckTemplateSettablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/CheckTemplateSettablePlan.java index df6a44cb83f55..42f4c30c7751d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/CheckTemplateSettablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/CheckTemplateSettablePlan.java @@ -19,25 +19,15 @@ package org.apache.iotdb.confignode.consensus.request.read.template; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; +public class CheckTemplateSettablePlan extends ConfigPhysicalReadPlan { -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; + private final String name; + private final String path; -public class CheckTemplateSettablePlan extends ConfigPhysicalPlan { - - private String name; - private String path; - - public CheckTemplateSettablePlan() { - super(ConfigPhysicalPlanType.CheckTemplateSettable); - } - - public CheckTemplateSettablePlan(String name, String path) { + public CheckTemplateSettablePlan(final String name, final String path) { super(ConfigPhysicalPlanType.CheckTemplateSettable); this.name = name; this.path = path; @@ -50,17 +40,4 @@ public String getName() { public String getPath() { return path; } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(name, stream); - ReadWriteIOUtils.write(path, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.name = ReadWriteIOUtils.readString(buffer); - this.path = ReadWriteIOUtils.readString(buffer); - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllSchemaTemplatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllSchemaTemplatePlan.java index bf1a7d8ff26ae..0add0c2edf795 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllSchemaTemplatePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllSchemaTemplatePlan.java @@ -19,24 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.template; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class GetAllSchemaTemplatePlan extends ConfigPhysicalPlan { +public class GetAllSchemaTemplatePlan extends ConfigPhysicalReadPlan { public GetAllSchemaTemplatePlan() { super(ConfigPhysicalPlanType.GetAllSchemaTemplate); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException {} } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllTemplateSetInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllTemplateSetInfoPlan.java index f13e5b02e78f9..c56f79165b838 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllTemplateSetInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetAllTemplateSetInfoPlan.java @@ -19,24 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.template; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class GetAllTemplateSetInfoPlan extends ConfigPhysicalPlan { +public class GetAllTemplateSetInfoPlan extends ConfigPhysicalReadPlan { public GetAllTemplateSetInfoPlan() { super(ConfigPhysicalPlanType.GetAllTemplateSetInfo); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException {} } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetPathsSetTemplatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetPathsSetTemplatePlan.java index e96d4d7957a77..f81805066af37 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetPathsSetTemplatePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetPathsSetTemplatePlan.java @@ -20,25 +20,15 @@ package org.apache.iotdb.confignode.consensus.request.read.template; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; +public class GetPathsSetTemplatePlan extends ConfigPhysicalReadPlan { -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; + private final String name; + private final PathPatternTree scope; -public class GetPathsSetTemplatePlan extends ConfigPhysicalPlan { - - private String name; - private PathPatternTree scope; - - public GetPathsSetTemplatePlan() { - super(ConfigPhysicalPlanType.GetPathsSetTemplate); - } - - public GetPathsSetTemplatePlan(String name, PathPatternTree scope) { + public GetPathsSetTemplatePlan(final String name, final PathPatternTree scope) { super(ConfigPhysicalPlanType.GetPathsSetTemplate); this.name = name; this.scope = scope; @@ -51,17 +41,4 @@ public String getName() { public PathPatternTree getScope() { return scope; } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(name, stream); - scope.serialize(stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.name = ReadWriteIOUtils.readString(buffer); - this.scope = PathPatternTree.deserialize(buffer); - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetSchemaTemplatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetSchemaTemplatePlan.java index 627dfa98c68d7..9545429923913 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetSchemaTemplatePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetSchemaTemplatePlan.java @@ -19,26 +19,17 @@ package org.apache.iotdb.confignode.consensus.request.read.template; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetSchemaTemplatePlan extends ConfigPhysicalPlan { +public class GetSchemaTemplatePlan extends ConfigPhysicalReadPlan { - private String templateName; + private final String templateName; - public GetSchemaTemplatePlan() { + public GetSchemaTemplatePlan(final String templateName) { super(ConfigPhysicalPlanType.GetSchemaTemplate); - } - - public GetSchemaTemplatePlan(String templateName) { - this(); this.templateName = templateName; } @@ -47,25 +38,14 @@ public String getTemplateName() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(getType().getPlanType(), stream); - ReadWriteIOUtils.write(templateName, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.templateName = ReadWriteIOUtils.readString(buffer); - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - GetSchemaTemplatePlan that = (GetSchemaTemplatePlan) o; + final GetSchemaTemplatePlan that = (GetSchemaTemplatePlan) o; return this.templateName.equalsIgnoreCase(that.templateName); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetTemplateSetInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetTemplateSetInfoPlan.java index a63e5ca912edc..4ee8ebd7afd9a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetTemplateSetInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/template/GetTemplateSetInfoPlan.java @@ -20,27 +20,16 @@ package org.apache.iotdb.confignode.consensus.request.read.template; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathDeserializeUtil; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; -public class GetTemplateSetInfoPlan extends ConfigPhysicalPlan { - - private List patternList; +public class GetTemplateSetInfoPlan extends ConfigPhysicalReadPlan { - public GetTemplateSetInfoPlan() { - super(ConfigPhysicalPlanType.GetTemplateSetInfo); - } + private final List patternList; - public GetTemplateSetInfoPlan(List patternList) { + public GetTemplateSetInfoPlan(final List patternList) { super(ConfigPhysicalPlanType.GetTemplateSetInfo); this.patternList = patternList; } @@ -48,22 +37,4 @@ public GetTemplateSetInfoPlan(List patternList) { public List getPatternList() { return patternList; } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(getType().getPlanType(), stream); - ReadWriteIOUtils.write(patternList.size(), stream); - for (PartialPath pattern : patternList) { - pattern.serialize(stream); - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int size = ReadWriteIOUtils.readInt(buffer); - patternList = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - patternList.add((PartialPath) PathDeserializeUtil.deserialize(buffer)); - } - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTransferringTriggersPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTransferringTriggersPlan.java index f7c9bf599e9de..d919ffd40ba1e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTransferringTriggersPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTransferringTriggersPlan.java @@ -19,26 +19,12 @@ package org.apache.iotdb.confignode.consensus.request.read.trigger; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class GetTransferringTriggersPlan extends ConfigPhysicalPlan { +public class GetTransferringTriggersPlan extends ConfigPhysicalReadPlan { public GetTransferringTriggersPlan() { super(ConfigPhysicalPlanType.GetTransferringTriggers); } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // do nothing - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerJarPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerJarPlan.java index 09addc05b859d..04ded3f97e5cd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerJarPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerJarPlan.java @@ -19,27 +19,17 @@ package org.apache.iotdb.confignode.consensus.request.read.trigger; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import java.util.Objects; -public class GetTriggerJarPlan extends ConfigPhysicalPlan { - - private List jarNames; +public class GetTriggerJarPlan extends ConfigPhysicalReadPlan { - public GetTriggerJarPlan() { - super(ConfigPhysicalPlanType.GetTriggerJar); - } + private final List jarNames; - public GetTriggerJarPlan(List triggerNames) { + public GetTriggerJarPlan(final List triggerNames) { super(ConfigPhysicalPlanType.GetTriggerJar); jarNames = triggerNames; } @@ -49,27 +39,7 @@ public List getJarNames() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(jarNames.size(), stream); - for (String jarName : jarNames) { - ReadWriteIOUtils.write(jarName, stream); - } - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int size = ReadWriteIOUtils.readInt(buffer); - jarNames = new ArrayList<>(size); - while (size > 0) { - jarNames.add(ReadWriteIOUtils.readString(buffer)); - size--; - } - } - - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } @@ -79,7 +49,7 @@ public boolean equals(Object o) { if (!super.equals(o)) { return false; } - GetTriggerJarPlan that = (GetTriggerJarPlan) o; + final GetTriggerJarPlan that = (GetTriggerJarPlan) o; return Objects.equals(jarNames, that.jarNames); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerLocationPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerLocationPlan.java index 597392519de13..a3dad77c035ea 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerLocationPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerLocationPlan.java @@ -19,26 +19,16 @@ package org.apache.iotdb.confignode.consensus.request.read.trigger; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetTriggerLocationPlan extends ConfigPhysicalPlan { - - String triggerName; +public class GetTriggerLocationPlan extends ConfigPhysicalReadPlan { + private final String triggerName; - public GetTriggerLocationPlan() { + public GetTriggerLocationPlan(final String triggerName) { super(ConfigPhysicalPlanType.GetTriggerLocation); - } - - public GetTriggerLocationPlan(String triggerName) { - this(); this.triggerName = triggerName; } @@ -46,24 +36,8 @@ public String getTriggerName() { return triggerName; } - public void setTriggerName(String triggerName) { - this.triggerName = triggerName; - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(triggerName, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.triggerName = ReadWriteIOUtils.readString(buffer); - } - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } @@ -73,7 +47,7 @@ public boolean equals(Object o) { if (!super.equals(o)) { return false; } - GetTriggerLocationPlan that = (GetTriggerLocationPlan) o; + final GetTriggerLocationPlan that = (GetTriggerLocationPlan) o; return Objects.equals(triggerName, that.triggerName); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerTablePlan.java index a1af8644ce01c..4388d78da2488 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/trigger/GetTriggerTablePlan.java @@ -19,26 +19,16 @@ package org.apache.iotdb.confignode.consensus.request.read.trigger; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Objects; -public class GetTriggerTablePlan extends ConfigPhysicalPlan { - - boolean onlyStateful; +public class GetTriggerTablePlan extends ConfigPhysicalReadPlan { + private final boolean onlyStateful; - public GetTriggerTablePlan() { + public GetTriggerTablePlan(final boolean onlyStateful) { super(ConfigPhysicalPlanType.GetTriggerTable); - } - - public GetTriggerTablePlan(boolean onlyStateful) { - this(); this.onlyStateful = onlyStateful; } @@ -46,24 +36,8 @@ public boolean isOnlyStateful() { return onlyStateful; } - public void setOnlyStateful(boolean onlyStateful) { - this.onlyStateful = onlyStateful; - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(onlyStateful, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.onlyStateful = ReadWriteIOUtils.readBool(buffer); - } - @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } @@ -73,7 +47,7 @@ public boolean equals(Object o) { if (!super.equals(o)) { return false; } - GetTriggerTablePlan that = (GetTriggerTablePlan) o; + final GetTriggerTablePlan that = (GetTriggerTablePlan) o; return onlyStateful == that.onlyStateful; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java index 9dd9eded31eb8..9f2992eeeb1ac 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java @@ -16,18 +16,15 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.confignode.consensus.request.read.ttl; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.db.utils.constant.SqlConstant; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class ShowTTLPlan extends ConfigPhysicalPlan { - private String[] pathPattern; +public class ShowTTLPlan extends ConfigPhysicalReadPlan { + private final String[] pathPattern; public String[] getPathPattern() { return pathPattern; @@ -38,14 +35,8 @@ public ShowTTLPlan() { this.pathPattern = SqlConstant.getSingleRootArray(); } - public ShowTTLPlan(String[] pathPattern) { + public ShowTTLPlan(final String[] pathPattern) { super(ConfigPhysicalPlanType.ShowTTL); this.pathPattern = pathPattern; } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException {} - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException {} } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/RegisterAINodePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/RegisterAINodePlan.java new file mode 100644 index 0000000000000..5f5cb9ae1ecf5 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/RegisterAINodePlan.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.ainode; + +import org.apache.iotdb.common.rpc.thrift.TAINodeConfiguration; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class RegisterAINodePlan extends ConfigPhysicalPlan { + + private TAINodeConfiguration aiNodeConfiguration; + + public RegisterAINodePlan() { + super(ConfigPhysicalPlanType.RegisterAINode); + } + + public RegisterAINodePlan(TAINodeConfiguration aiNodeConfiguration) { + this(); + this.aiNodeConfiguration = aiNodeConfiguration; + } + + public TAINodeConfiguration getAINodeConfiguration() { + return aiNodeConfiguration; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ThriftCommonsSerDeUtils.serializeTAINodeInfo(aiNodeConfiguration, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + aiNodeConfiguration = ThriftCommonsSerDeUtils.deserializeTAINodeInfo(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RegisterAINodePlan that = (RegisterAINodePlan) o; + return aiNodeConfiguration.equals(that.aiNodeConfiguration); + } + + @Override + public int hashCode() { + return Objects.hash(aiNodeConfiguration); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/RemoveAINodePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/RemoveAINodePlan.java new file mode 100644 index 0000000000000..92bfb8b7017fa --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/RemoveAINodePlan.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.ainode; + +import org.apache.iotdb.common.rpc.thrift.TAINodeLocation; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class RemoveAINodePlan extends ConfigPhysicalPlan { + + private TAINodeLocation aiNodeLocation; + + public RemoveAINodePlan() { + super(ConfigPhysicalPlanType.RemoveAINode); + } + + public RemoveAINodePlan(TAINodeLocation taiNodeLocation) { + this(); + this.aiNodeLocation = taiNodeLocation; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ThriftCommonsSerDeUtils.serializeTAINodeLocation(aiNodeLocation, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.aiNodeLocation = ThriftCommonsSerDeUtils.deserializeTAINodeLocation(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + RemoveAINodePlan that = (RemoveAINodePlan) o; + return aiNodeLocation.equals(that.aiNodeLocation); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), aiNodeLocation); + } + + public TAINodeLocation getAINodeLocation() { + return aiNodeLocation; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/UpdateAINodePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/UpdateAINodePlan.java new file mode 100644 index 0000000000000..5ef885551d7ab --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/ainode/UpdateAINodePlan.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.ainode; + +import org.apache.iotdb.common.rpc.thrift.TAINodeConfiguration; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class UpdateAINodePlan extends ConfigPhysicalPlan { + + private TAINodeConfiguration aiNodeConfiguration; + + public UpdateAINodePlan() { + super(ConfigPhysicalPlanType.UpdateAINodeConfiguration); + } + + public UpdateAINodePlan(TAINodeConfiguration aiNodeConfiguration) { + this(); + this.aiNodeConfiguration = aiNodeConfiguration; + } + + public TAINodeConfiguration getAINodeConfiguration() { + return aiNodeConfiguration; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ThriftCommonsSerDeUtils.serializeTAINodeConfiguration(aiNodeConfiguration, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + aiNodeConfiguration = ThriftCommonsSerDeUtils.deserializeTAINodeConfiguration(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!getType().equals(((UpdateAINodePlan) o).getType())) { + return false; + } + UpdateAINodePlan that = (UpdateAINodePlan) o; + return aiNodeConfiguration.equals(that.aiNodeConfiguration); + } + + @Override + public int hashCode() { + return Objects.hash(getType(), aiNodeConfiguration); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorPlan.java new file mode 100644 index 0000000000000..5c78c49813c47 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorPlan.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.auth; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +import java.util.Objects; + +public abstract class AuthorPlan extends ConfigPhysicalReadPlan { + + protected String roleName; + protected String password; + protected String newPassword; + protected String userName; + protected boolean grantOpt; + + public AuthorPlan(final ConfigPhysicalPlanType type) { + super(type); + } + + // To maintain compatibility between TreeAuthorPlan and RelationalAuthorPlan, + // the newPassword field is retained here. + public AuthorPlan( + ConfigPhysicalPlanType type, + String userName, + String roleName, + String password, + String newPassword, + boolean grantOpt) { + super(type); + this.userName = userName; + this.roleName = roleName; + this.password = password; + this.newPassword = newPassword; + this.grantOpt = grantOpt; + } + + public ConfigPhysicalPlanType getAuthorType() { + return super.getType(); + } + + public String getRoleName() { + return roleName; + } + + public void setRoleName(final String roleName) { + this.roleName = roleName; + } + + public String getPassword() { + return password; + } + + public void setPassword(final String password) { + this.password = password; + } + + public String getNewPassword() { + return newPassword; + } + + public void setNewPassword(String newPassword) { + this.newPassword = newPassword; + } + + public boolean getGrantOpt() { + return this.grantOpt; + } + + public void setGrantOpt(final boolean grantOpt) { + this.grantOpt = grantOpt; + } + + public String getUserName() { + return userName; + } + + public void setUserName(final String userName) { + this.userName = userName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AuthorPlan)) { + return false; + } + AuthorPlan that = (AuthorPlan) o; + return Objects.equals(super.getType(), that.getAuthorType()) + && Objects.equals(userName, that.userName) + && Objects.equals(roleName, that.roleName) + && Objects.equals(password, that.password) + && Objects.equals(newPassword, that.newPassword) + && grantOpt == that.grantOpt; + } + + @Override + public int hashCode() { + return Objects.hash(super.getType(), userName, roleName, password, newPassword, grantOpt); + } + + @Override + public String toString() { + return "[type:" + + super.getType() + + ", username:" + + userName + + ", rolename:" + + roleName + + ", grant option:" + + grantOpt + + "]"; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorRelationalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorRelationalPlan.java new file mode 100644 index 0000000000000..87c38d781dddb --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorRelationalPlan.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.consensus.request.write.auth; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +public class AuthorRelationalPlan extends AuthorPlan { + protected Set permissions; + protected String databaseName; + protected String tableName; + + public AuthorRelationalPlan(final ConfigPhysicalPlanType authorType) { + super(authorType); + } + + public AuthorRelationalPlan( + final ConfigPhysicalPlanType authorType, + final String userName, + final String roleName, + final String databaseName, + final String tableName, + final Set permissions, + final boolean grantOpt, + final String password) { + super(authorType, userName, roleName, password, "", grantOpt); + this.databaseName = databaseName; + this.tableName = tableName; + this.permissions = permissions; + } + + public AuthorRelationalPlan( + final ConfigPhysicalPlanType authorType, + final String userName, + final String roleName, + final String databaseName, + final String tableName, + final int permission, + final boolean grantOpt) { + super(authorType, userName, roleName, "", "", grantOpt); + this.databaseName = databaseName; + this.tableName = tableName; + this.permissions = Collections.singleton(permission); + } + + public String getDatabaseName() { + return databaseName; + } + + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public Set getPermissions() { + return permissions; + } + + public void setPermissions(Set permissions) { + this.permissions = permissions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof AuthorRelationalPlan) { + AuthorRelationalPlan that = (AuthorRelationalPlan) o; + return super.equals(o) + && Objects.equals(this.databaseName, that.databaseName) + && Objects.equals(this.tableName, that.tableName) + && Objects.equals(this.permissions, that.permissions); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), databaseName, tableName, permissions); + } + + @Override + public String toString() { + return "[type:" + + super.getType() + + ", name:" + + userName + + ", role:" + + roleName + + ", permissions:" + + PrivilegeType.toPriType(permissions) + + ", grant option:" + + grantOpt + + ", DB:" + + databaseName + + ", TABLE:" + + tableName + + "]"; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(getType().getPlanType(), stream); + BasicStructureSerDeUtil.write(userName, stream); + BasicStructureSerDeUtil.write(roleName, stream); + BasicStructureSerDeUtil.write(password, stream); + BasicStructureSerDeUtil.write(databaseName, stream); + BasicStructureSerDeUtil.write(tableName, stream); + stream.writeInt(permissions.size()); + for (Integer permission : permissions) { + stream.writeInt(permission); + } + stream.write(grantOpt ? (byte) 1 : (byte) 0); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) { + newPassword = ""; + userName = BasicStructureSerDeUtil.readString(buffer); + roleName = BasicStructureSerDeUtil.readString(buffer); + password = BasicStructureSerDeUtil.readString(buffer); + databaseName = BasicStructureSerDeUtil.readString(buffer); + tableName = BasicStructureSerDeUtil.readString(buffer); + permissions = new HashSet<>(); + int size = buffer.getInt(); + for (int i = 0; i < size; i++) { + permissions.add(buffer.getInt()); + } + grantOpt = buffer.get() == (byte) 1; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorTreePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorTreePlan.java new file mode 100644 index 0000000000000..25c7ec92cd6ed --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/auth/AuthorTreePlan.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.consensus.request.write.auth; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +public class AuthorTreePlan extends AuthorPlan { + protected Set permissions; + protected List nodeNameList; + + public AuthorTreePlan(final ConfigPhysicalPlanType type) { + super(type); + } + + /** + * {@link AuthorTreePlan} Constructor. + * + * @param authorType author type + * @param userName user name + * @param roleName role name + * @param password password + * @param permissions permissions + * @param grantOpt with grant option, only grant statement can set grantOpt = true + * @param nodeNameList node name in Path structure + */ + public AuthorTreePlan( + final ConfigPhysicalPlanType authorType, + final String userName, + final String roleName, + final String password, + final String newPassword, + final Set permissions, + final boolean grantOpt, + final List nodeNameList) { + super(authorType, userName, roleName, password, newPassword, grantOpt); + this.permissions = permissions; + this.nodeNameList = nodeNameList; + } + + public Set getPermissions() { + return permissions; + } + + public void setPermissions(Set permissions) { + this.permissions = permissions; + } + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof AuthorTreePlan) { + AuthorTreePlan that = (AuthorTreePlan) o; + return super.equals(that) + && Objects.equals(permissions, that.permissions) + && Objects.equals(nodeNameList, that.nodeNameList); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), permissions, nodeNameList); + } + + @Override + public String toString() { + return "[type:" + + super.getType() + + ", username:" + + super.getUserName() + + ", rolename:" + + super.getRoleName() + + ", permissions:" + + PrivilegeType.toPriType(permissions) + + ", grant option:" + + super.getGrantOpt() + + ", paths:" + + nodeNameList + + "]"; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(getType().getPlanType(), stream); + BasicStructureSerDeUtil.write(userName, stream); + BasicStructureSerDeUtil.write(roleName, stream); + BasicStructureSerDeUtil.write(password, stream); + BasicStructureSerDeUtil.write(newPassword, stream); + if (permissions == null) { + stream.write((byte) 0); + } else { + stream.write((byte) 1); + stream.writeInt(permissions.size()); + for (int permission : permissions) { + stream.writeInt(permission); + } + } + BasicStructureSerDeUtil.write(nodeNameList.size(), stream); + for (PartialPath partialPath : nodeNameList) { + BasicStructureSerDeUtil.write(partialPath.getFullPath(), stream); + } + BasicStructureSerDeUtil.write(super.getGrantOpt() ? 1 : 0, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) { + userName = BasicStructureSerDeUtil.readString(buffer); + roleName = BasicStructureSerDeUtil.readString(buffer); + password = BasicStructureSerDeUtil.readString(buffer); + newPassword = BasicStructureSerDeUtil.readString(buffer); + if (buffer.get() == (byte) 0) { + this.permissions = null; + } else { + int permissionsSize = buffer.getInt(); + this.permissions = new HashSet<>(); + for (int i = 0; i < permissionsSize; i++) { + permissions.add(buffer.getInt()); + } + } + + int nodeNameListSize = BasicStructureSerDeUtil.readInt(buffer); + nodeNameList = new ArrayList<>(nodeNameListSize); + try { + for (int i = 0; i < nodeNameListSize; i++) { + nodeNameList.add(new PartialPath(BasicStructureSerDeUtil.readString(buffer))); + } + } catch (MetadataException e) { + // do nothing + } + if (super.getAuthorType().ordinal() >= ConfigPhysicalPlanType.CreateUser.ordinal()) { + super.setGrantOpt(BasicStructureSerDeUtil.readInt(buffer) > 0); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ShowCQPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ShowCQPlan.java deleted file mode 100644 index 7c146238a5ed9..0000000000000 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ShowCQPlan.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.consensus.request.write.cq; - -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.SHOW_CQ; - -public class ShowCQPlan extends ConfigPhysicalPlan { - - public ShowCQPlan() { - super(SHOW_CQ); - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - // no customized field to deserialize from - } -} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/AdjustMaxRegionGroupNumPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/AdjustMaxRegionGroupNumPlan.java index c6dbd600278a4..42b98cc30d03b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/AdjustMaxRegionGroupNumPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/AdjustMaxRegionGroupNumPlan.java @@ -42,7 +42,7 @@ public AdjustMaxRegionGroupNumPlan() { this.maxRegionGroupNumMap = new HashMap<>(); } - public void putEntry(String storageGroup, Pair maxRegionGroupNum) { + public void putEntry(final String storageGroup, final Pair maxRegionGroupNum) { maxRegionGroupNumMap.put(storageGroup, maxRegionGroupNum); } @@ -51,11 +51,11 @@ public Map> getMaxRegionGroupNumMap() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { + protected void serializeImpl(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(getType().getPlanType(), stream); ReadWriteIOUtils.write(maxRegionGroupNumMap.size(), stream); - for (Map.Entry> maxRegionGroupNumEntry : + for (final Map.Entry> maxRegionGroupNumEntry : maxRegionGroupNumMap.entrySet()) { ReadWriteIOUtils.write(maxRegionGroupNumEntry.getKey(), stream); ReadWriteIOUtils.write(maxRegionGroupNumEntry.getValue().getLeft(), stream); @@ -64,27 +64,27 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { } @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int storageGroupNum = buffer.getInt(); + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + final int storageGroupNum = buffer.getInt(); for (int i = 0; i < storageGroupNum; i++) { - String storageGroup = ReadWriteIOUtils.readString(buffer); - int maxSchemaRegionGroupNum = buffer.getInt(); - int maxDataRegionGroupNum = buffer.getInt(); + final String storageGroup = ReadWriteIOUtils.readString(buffer); + final int maxSchemaRegionGroupNum = buffer.getInt(); + final int maxDataRegionGroupNum = buffer.getInt(); maxRegionGroupNumMap.put( storageGroup, new Pair<>(maxSchemaRegionGroupNum, maxDataRegionGroupNum)); } } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - AdjustMaxRegionGroupNumPlan that = (AdjustMaxRegionGroupNumPlan) o; + final AdjustMaxRegionGroupNumPlan that = (AdjustMaxRegionGroupNumPlan) o; return maxRegionGroupNumMap.equals(that.maxRegionGroupNumMap); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/DatabaseSchemaPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/DatabaseSchemaPlan.java index 96910a8e92502..b801576215048 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/DatabaseSchemaPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/database/DatabaseSchemaPlan.java @@ -33,12 +33,12 @@ public class DatabaseSchemaPlan extends ConfigPhysicalPlan { private TDatabaseSchema schema; - public DatabaseSchemaPlan(ConfigPhysicalPlanType planType) { + public DatabaseSchemaPlan(final ConfigPhysicalPlanType planType) { super(planType); this.schema = new TDatabaseSchema(); } - public DatabaseSchemaPlan(ConfigPhysicalPlanType planType, TDatabaseSchema schema) { + public DatabaseSchemaPlan(final ConfigPhysicalPlanType planType, final TDatabaseSchema schema) { this(planType); this.schema = schema; } @@ -48,25 +48,25 @@ public TDatabaseSchema getSchema() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { + protected void serializeImpl(final DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ThriftConfigNodeSerDeUtils.serializeTDatabaseSchema(schema, stream); } @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { schema = ThriftConfigNodeSerDeUtils.deserializeTDatabaseSchema(buffer); } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - DatabaseSchemaPlan that = (DatabaseSchemaPlan) o; + final DatabaseSchemaPlan that = (DatabaseSchemaPlan) o; return schema.equals(that.schema); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropFunctionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropFunctionPlan.java deleted file mode 100644 index a4ecc6330b30e..0000000000000 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropFunctionPlan.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.consensus.request.write.function; - -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; - -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Objects; - -public class DropFunctionPlan extends ConfigPhysicalPlan { - - private String functionName; - - public DropFunctionPlan() { - super(ConfigPhysicalPlanType.DropFunction); - } - - public DropFunctionPlan(String functionName) { - super(ConfigPhysicalPlanType.DropFunction); - this.functionName = functionName; - } - - public String getFunctionName() { - return functionName; - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(functionName, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - functionName = ReadWriteIOUtils.readString(buffer); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - DropFunctionPlan that = (DropFunctionPlan) o; - return Objects.equals(functionName, that.functionName); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), functionName); - } -} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropTableModelFunctionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropTableModelFunctionPlan.java new file mode 100644 index 0000000000000..f43f9339bbaba --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropTableModelFunctionPlan.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.function; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DropTableModelFunctionPlan extends ConfigPhysicalPlan { + + private String functionName; + + public DropTableModelFunctionPlan() { + super(ConfigPhysicalPlanType.DropTableModelFunction); + } + + public DropTableModelFunctionPlan(String functionName) { + super(ConfigPhysicalPlanType.DropTableModelFunction); + this.functionName = functionName; + } + + public String getFunctionName() { + return functionName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(functionName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + functionName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + DropTableModelFunctionPlan that = (DropTableModelFunctionPlan) o; + return Objects.equals(functionName, that.functionName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), functionName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropTreeModelFunctionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropTreeModelFunctionPlan.java new file mode 100644 index 0000000000000..ff13ed03b3fe3 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/DropTreeModelFunctionPlan.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.function; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DropTreeModelFunctionPlan extends ConfigPhysicalPlan { + + private String functionName; + + public DropTreeModelFunctionPlan() { + super(ConfigPhysicalPlanType.DropTreeModelFunction); + } + + public DropTreeModelFunctionPlan(String functionName) { + super(ConfigPhysicalPlanType.DropTreeModelFunction); + this.functionName = functionName; + } + + public String getFunctionName() { + return functionName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(functionName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + functionName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + DropTreeModelFunctionPlan that = (DropTreeModelFunctionPlan) o; + return Objects.equals(functionName, that.functionName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), functionName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/UpdateFunctionPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/UpdateFunctionPlan.java new file mode 100644 index 0000000000000..eb91ae678194f --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/function/UpdateFunctionPlan.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.function; + +import org.apache.iotdb.commons.udf.UDFInformation; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class UpdateFunctionPlan extends ConfigPhysicalPlan { + private UDFInformation udfInformation; + + public UpdateFunctionPlan() { + super(ConfigPhysicalPlanType.UpdateFunction); + } + + public UpdateFunctionPlan(UDFInformation udfInformation) { + super(ConfigPhysicalPlanType.UpdateFunction); + this.udfInformation = udfInformation; + } + + public UDFInformation getUdfInformation() { + return udfInformation; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + udfInformation.serialize(stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + udfInformation = UDFInformation.deserialize(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + UpdateFunctionPlan that = (UpdateFunctionPlan) o; + return Objects.equals(udfInformation, that.udfInformation); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), udfInformation); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java new file mode 100644 index 0000000000000..61e37cdd21877 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.model; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class CreateModelPlan extends ConfigPhysicalPlan { + + private String modelName; + + public CreateModelPlan() { + super(ConfigPhysicalPlanType.CreateModel); + } + + public CreateModelPlan(String modelName) { + super(ConfigPhysicalPlanType.CreateModel); + this.modelName = modelName; + } + + public String getModelName() { + return modelName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(modelName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + modelName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + CreateModelPlan that = (CreateModelPlan) o; + return Objects.equals(modelName, that.modelName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), modelName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelInNodePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelInNodePlan.java new file mode 100644 index 0000000000000..885543f84e156 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelInNodePlan.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.model; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DropModelInNodePlan extends ConfigPhysicalPlan { + + private int nodeId; + + public DropModelInNodePlan() { + super(ConfigPhysicalPlanType.DropModelInNode); + } + + public DropModelInNodePlan(int nodeId) { + super(ConfigPhysicalPlanType.DropModelInNode); + this.nodeId = nodeId; + } + + public int getNodeId() { + return nodeId; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + stream.writeInt(nodeId); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + nodeId = buffer.getInt(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DropModelInNodePlan)) return false; + DropModelInNodePlan that = (DropModelInNodePlan) o; + return nodeId == that.nodeId; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), nodeId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java new file mode 100644 index 0000000000000..813b116c645c5 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.model; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DropModelPlan extends ConfigPhysicalPlan { + + private String modelName; + + public DropModelPlan() { + super(ConfigPhysicalPlanType.DropModel); + } + + public DropModelPlan(String modelName) { + super(ConfigPhysicalPlanType.DropModel); + this.modelName = modelName; + } + + public String getModelName() { + return modelName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(modelName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + modelName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + DropModelPlan that = (DropModelPlan) o; + return modelName.equals(that.modelName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), modelName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java new file mode 100644 index 0000000000000..ce7219e428139 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.model; + +import org.apache.iotdb.commons.model.ModelInformation; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class UpdateModelInfoPlan extends ConfigPhysicalPlan { + + private String modelName; + private ModelInformation modelInformation; + + // The node which has the model which is only updated in model registration + private List nodeIds; + + public UpdateModelInfoPlan() { + super(ConfigPhysicalPlanType.UpdateModelInfo); + } + + public UpdateModelInfoPlan(String modelName, ModelInformation modelInformation) { + super(ConfigPhysicalPlanType.UpdateModelInfo); + this.modelName = modelName; + this.modelInformation = modelInformation; + this.nodeIds = Collections.emptyList(); + } + + public UpdateModelInfoPlan( + String modelName, ModelInformation modelInformation, List nodeIds) { + super(ConfigPhysicalPlanType.UpdateModelInfo); + this.modelName = modelName; + this.modelInformation = modelInformation; + this.nodeIds = nodeIds; + } + + public String getModelName() { + return modelName; + } + + public ModelInformation getModelInformation() { + return modelInformation; + } + + public List getNodeIds() { + return nodeIds; + } + + public void setNodeIds(List nodeIds) { + this.nodeIds = nodeIds; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(modelName, stream); + this.modelInformation.serialize(stream); + ReadWriteIOUtils.write(nodeIds.size(), stream); + for (Integer nodeId : nodeIds) { + ReadWriteIOUtils.write(nodeId, stream); + } + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.modelName = ReadWriteIOUtils.readString(buffer); + this.modelInformation = ModelInformation.deserialize(buffer); + int size = ReadWriteIOUtils.readInt(buffer); + this.nodeIds = new ArrayList<>(); + for (int i = 0; i < size; i++) { + this.nodeIds.add(ReadWriteIOUtils.readInt(buffer)); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + UpdateModelInfoPlan that = (UpdateModelInfoPlan) o; + return modelName.equals(that.modelName) + && modelInformation.equals(that.modelInformation) + && nodeIds.equals(that.nodeIds); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), modelName, modelInformation, nodeIds); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/AutoCleanPartitionTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/AutoCleanPartitionTablePlan.java new file mode 100644 index 0000000000000..1b1bfbe38f029 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/AutoCleanPartitionTablePlan.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.partition; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +public class AutoCleanPartitionTablePlan extends ConfigPhysicalPlan { + + Map databaseTTLMap; + TTimePartitionSlot currentTimeSlot; + + public AutoCleanPartitionTablePlan() { + super(ConfigPhysicalPlanType.AutoCleanPartitionTable); + } + + public AutoCleanPartitionTablePlan( + Map databaseTTLMap, TTimePartitionSlot currentTimeSlot) { + this(); + this.databaseTTLMap = databaseTTLMap; + this.currentTimeSlot = currentTimeSlot; + } + + public Map getDatabaseTTLMap() { + return databaseTTLMap; + } + + public TTimePartitionSlot getCurrentTimeSlot() { + return currentTimeSlot; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + stream.writeInt(databaseTTLMap.size()); + for (Map.Entry entry : databaseTTLMap.entrySet()) { + BasicStructureSerDeUtil.write(entry.getKey(), stream); + stream.writeLong(entry.getValue()); + } + ThriftCommonsSerDeUtils.serializeTTimePartitionSlot(currentTimeSlot, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + int size = buffer.getInt(); + databaseTTLMap = new TreeMap<>(); + for (int i = 0; i < size; i++) { + String key = BasicStructureSerDeUtil.readString(buffer); + long value = buffer.getLong(); + databaseTTLMap.put(key, value); + } + currentTimeSlot = ThriftCommonsSerDeUtils.deserializeTTimePartitionSlot(buffer); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + AutoCleanPartitionTablePlan that = (AutoCleanPartitionTablePlan) o; + return Objects.equals(databaseTTLMap, that.databaseTTLMap) + && Objects.equals(currentTimeSlot, that.currentTimeSlot); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), databaseTTLMap, currentTimeSlot); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/RemoveRegionLocationPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/RemoveRegionLocationPlan.java index 9dc6a00705289..47bc6493aa169 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/RemoveRegionLocationPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/RemoveRegionLocationPlan.java @@ -58,11 +58,6 @@ protected void deserializeImpl(ByteBuffer buffer) throws IOException { deprecatedLocation = ThriftCommonsSerDeUtils.deserializeTDataNodeLocation(buffer); } - @Override - public ConfigPhysicalPlanType getType() { - return super.getType(); - } - public TConsensusGroupId getRegionId() { return regionId; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeCreateTableOrViewPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeCreateTableOrViewPlan.java new file mode 100644 index 0000000000000..65285d2190804 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeCreateTableOrViewPlan.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.pipe.payload; + +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; + +public class PipeCreateTableOrViewPlan extends PreCreateTablePlan { + public PipeCreateTableOrViewPlan() { + super(ConfigPhysicalPlanType.PipeCreateTableOrView); + } + + public PipeCreateTableOrViewPlan(final String database, final TsTable table) { + super(ConfigPhysicalPlanType.PipeCreateTableOrView, database, table); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java index cf9afa0fd8c9d..b91d7ab8c88d8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java @@ -44,7 +44,7 @@ public PipeDeactivateTemplatePlan() { super(ConfigPhysicalPlanType.PipeDeactivateTemplate); } - public PipeDeactivateTemplatePlan(Map> templateSetInfo) { + public PipeDeactivateTemplatePlan(final Map> templateSetInfo) { super(ConfigPhysicalPlanType.PipeDeactivateTemplate); this.templateSetInfo = templateSetInfo; } @@ -54,26 +54,26 @@ public Map> getTemplateSetInfo() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { + protected void serializeImpl(final DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(templateSetInfo.size(), stream); - for (Map.Entry> entry : templateSetInfo.entrySet()) { + for (final Map.Entry> entry : templateSetInfo.entrySet()) { entry.getKey().serialize(stream); ReadWriteIOUtils.write(entry.getValue().size(), stream); - for (Template template : entry.getValue()) { + for (final Template template : entry.getValue()) { template.serialize(stream); } } } @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int size = ReadWriteIOUtils.readInt(buffer); + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + final int size = ReadWriteIOUtils.readInt(buffer); templateSetInfo = new HashMap<>(); for (int i = 0; i < size; i++) { - PartialPath pattern = (PartialPath) PathDeserializeUtil.deserialize(buffer); - int templateNum = ReadWriteIOUtils.readInt(buffer); - List